cnpg.log

{"level":"info","ts":"2025-08-11T10:23:06.328173949Z","msg":"Cluster status","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","currentPrimary":"cnpg-1","targetPrimary":"cnpg-1","isReplicaCluster":false}
{"level":"info","ts":"2025-08-11T10:23:06.33031657Z","msg":"This is an old primary instance, waiting for the switchover to finish","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","currentPrimary":"cnpg-1","targetPrimary":"cnpg-1"}
{"level":"info","ts":"2025-08-11T10:23:06.330486772Z","msg":"Switchover completed","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","targetPrimary":"cnpg-1","currentPrimary":"cnpg-1"}
{"level":"info","ts":"2025-08-11T10:23:06.330564873Z","msg":"Waiting for the new primary to be available","logging_pod":"cnpg-2","primaryConnInfo":"host=cnpg-rw user=streaming_replica port=5432 sslkey=/controller/certificates/streaming_replica.key sslcert=/controller/certificates/streaming_replica.crt sslrootcert=/controller/certificates/server-ca.crt application_name=cnpg-2 sslmode=verify-ca dbname=postgres connect_timeout=5"}
{"level":"info","ts":"2025-08-11T10:23:06.346577031Z","msg":"Extracting pg_controldata information","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","reason":"before pg_rewind"}
{"level":"info","ts":"2025-08-11T10:23:06.347971645Z","logger":"pg_controldata","msg":"pg_control version number: 1300\nCatalog version number: 202307071\nDatabase system identifier: 7421150433874227218\nDatabase cluster state: shut down\npg_control last modified: Mon 11 Aug 2025 08:43:27 AM UTC\nLatest checkpoint location: 2/5F000028\nLatest checkpoint's REDO location: 2/5F000028\nLatest checkpoint's REDO WAL file: 00000068000000020000005F\nLatest checkpoint's TimeLineID: 104\nLatest checkpoint's PrevTimeLineID: 104\nLatest checkpoint's full_page_writes: on\nLatest checkpoint's NextXID: 0:4641\nLatest checkpoint's NextOID: 205629\nLatest checkpoint's NextMultiXactId: 1\nLatest checkpoint's NextMultiOffset: 0\nLatest checkpoint's oldestXID: 722\nLatest checkpoint's oldestXID's DB: 1\nLatest checkpoint's oldestActiveXID: 0\nLatest checkpoint's oldestMultiXid: 1\nLatest checkpoint's oldestMulti's DB: 1\nLatest checkpoint's oldestCommitTsXid:0\nLatest checkpoint's newestCommitTsXid:0\nTime of latest checkpoint: Mon 11 Aug 2025 08:43:27 AM UTC\nFake LSN counter for unlogged rels: 0/3E8\nMinimum recovery ending location: 0/0\nMin recovery ending loc's timeline: 0\nBackup start location: 0/0\nBackup end location: 0/0\nEnd-of-backup record required: no\nwal_level setting: logical\nwal_log_hints setting: on\nmax_connections setting: 100\nmax_worker_processes setting: 32\nmax_wal_senders setting: 10\nmax_prepared_xacts setting: 0\nmax_locks_per_xact setting: 64\ntrack_commit_timestamp setting: off\nMaximum data alignment: 8\nDatabase block size: 8192\nBlocks per segment of large relation: 131072\nWAL block size: 8192\nBytes per WAL segment: 16777216\nMaximum length of identifiers: 64\nMaximum columns in an index: 32\nMaximum size of a TOAST chunk: 1996\nSize of a large-object chunk: 2048\nDate/time type storage: 64-bit integers\nFloat8 argument passing: by value\nData page checksum version: 1\nMock authentication nonce: d6c55c6c0f08940bd776ec30b5a39494fa902e9dda31ce8a661c9d68f3e144b5\n","pipe":"stdout","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:23:06.359204356Z","msg":"Starting up pg_rewind","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","pgdata":"/var/lib/postgresql/data/pgdata","options":["-P","--source-server","host=cnpg-rw user=streaming_replica port=5432 sslkey=/controller/certificates/streaming_replica.key sslcert=/controller/certificates/streaming_replica.crt sslrootcert=/controller/certificates/server-ca.crt application_name=cnpg-2 sslmode=verify-ca dbname=postgres","--target-pgdata","/var/lib/postgresql/data/pgdata","--restore-target-wal"]}
{"level":"info","ts":"2025-08-11T10:23:06.403206791Z","logger":"pg_rewind","msg":"pg_rewind: connected to server","pipe":"stderr","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:23:06.41516021Z","logger":"pg_rewind","msg":"pg_rewind: error: could not find common ancestor of the source and target cluster's timelines","pipe":"stderr","logging_pod":"cnpg-2"}
{"level":"error","ts":"2025-08-11T10:23:06.416164919Z","msg":"Failed to execute pg_rewind","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","logging_pod":"cnpg-2","options":["-P","--source-server","host=cnpg-rw user=streaming_replica port=5432 sslkey=/controller/certificates/streaming_replica.key sslcert=/controller/certificates/streaming_replica.crt sslrootcert=/controller/certificates/server-ca.crt application_name=cnpg-2 sslmode=verify-ca dbname=postgres","--target-pgdata","/var/lib/postgresql/data/pgdata","--restore-target-wal"],"error":"exit status 1","stacktrace":"github.com/cloudnative-pg/machinery/pkg/log.(*logger).Error\n\tpkg/mod/github.com/cloudnative-pg/machinery@v0.2.0/pkg/log/log.go:125\ngithub.com/cloudnative-pg/cloudnative-pg/pkg/management/postgres.(*Instance).Rewind\n\tpkg/management/postgres/instance.go:1069\ngithub.com/cloudnative-pg/cloudnative-pg/internal/management/controller.(*InstanceReconciler).verifyPgDataCoherenceForPrimary\n\tinternal/management/controller/instance_startup.go:133\ngithub.com/cloudnative-pg/cloudnative-pg/internal/management/controller.(*InstanceReconciler).initialize\n\tinternal/management/controller/instance_controller.go:416\ngithub.com/cloudnative-pg/cloudnative-pg/internal/management/controller.(*InstanceReconciler).Reconcile\n\tinternal/management/controller/instance_controller.go:173\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"error","ts":"2025-08-11T10:23:06.416394522Z","msg":"Reconciler error","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"cnpg","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg","reconcileID":"976148ef-3b32-4d00-98ae-862ff0822143","error":"while exucuting pg_rewind: error executing pg_rewind: exit status 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\tpkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2025-08-11T10:23:15.512959996Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"55b75057-eaaa-40f4-8c90-7d0fabd52944","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:23:15.512986696Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"55b75057-eaaa-40f4-8c90-7d0fabd52944","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:23:45.513995442Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"dcfd2505-0dd6-4c75-b3da-655c0c6b5ed4","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:23:45.514024042Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"dcfd2505-0dd6-4c75-b3da-655c0c6b5ed4","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:24:15.514903156Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"2a51489b-9f11-4300-b10e-08117b5e6dda","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:24:15.514928156Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"2a51489b-9f11-4300-b10e-08117b5e6dda","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:24:45.515755248Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"f5f64736-f4e8-4c47-b14b-7569ea96d141","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:24:45.515781748Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"f5f64736-f4e8-4c47-b14b-7569ea96d141","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:25:15.51720642Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"2cbedcec-259c-4e2b-8d91-c2820f21ab08","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:25:15.517528219Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"2cbedcec-259c-4e2b-8d91-c2820f21ab08","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:25:45.518821002Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"82782dc6-5ada-436f-9053-d7a3349f6e37","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:25:45.518853102Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"82782dc6-5ada-436f-9053-d7a3349f6e37","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:26:15.519145964Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"c59a21c2-3a83-4793-a7dc-36d2ddbafd7a","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:26:15.519174565Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"c59a21c2-3a83-4793-a7dc-36d2ddbafd7a","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:26:45.519920693Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"84298cd0-91e9-47c1-85b1-b3a5e47583c9","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:26:45.519946393Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"84298cd0-91e9-47c1-85b1-b3a5e47583c9","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:27:15.52036425Z","logger":"database_reconciler","msg":"Reconciling database","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"76a96820-5ae9-4379-a0df-f074c1196762","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}
{"level":"info","ts":"2025-08-11T10:27:15.52039165Z","logger":"database_reconciler","msg":"Reconciliation loop of database exited","logger":"instance-manager","logging_pod":"cnpg-2","controller":"instance-database","controllerGroup":"postgresql.cnpg.io","controllerKind":"Database","Database":{"name":"cnpg-test-db","namespace":"gorilla-test"},"namespace":"gorilla-test","name":"cnpg-test-db","reconcileID":"76a96820-5ae9-4379-a0df-f074c1196762","databaseName":"cnpg-test-db","logging_pod":"cnpg-2"}

Last two lines are duplicated over and over again.