+ cd /var/lib/postgresql/ADBDEV/5716.1 + test -f master/postmaster.pid + echo 1 1 + rm -rf master + mkdir -p master + pg_ctl initdb -D master -o '--data-checksums -N -A trust --wal-segsize 1' The files belonging to this database system will be owned by user "postgres". This user must also own the server process. The database cluster will be initialized with locale "ru_RU.UTF-8". The default database encoding has accordingly been set to "UTF8". The default text search configuration will be set to "russian". Data page checksums are enabled. fixing permissions on existing directory master ... ok creating subdirectories ... ok selecting dynamic shared memory implementation ... posix selecting default max_connections ... 100 selecting default shared_buffers ... 128MB selecting default time zone ... Asia/Yekaterinburg creating configuration files ... ok running bootstrap script ... ok performing post-bootstrap initialization ... ok Sync to disk skipped. The data directory might become corrupt if the operating system crashes. Success. You can now start the database server using: /usr/local/bin/pg_ctl -D master -l logfile start + cat + cat + pg_ctl -w -D master start -o '-p 5000' waiting for server to start....2024-08-08 14:36:05.716 +05 [2259] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:05.716 +05 [2259] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:05.719 +05 [2259] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:05.719 +05 [2259] DEBUG: created dynamic shared memory control segment 511288034 (26976 bytes) 2024-08-08 14:36:05.719 +05 [2259] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:05.720 +05 [2259] LOG: redirecting log output to logging collector process 2024-08-08 14:36:05.720 +05 [2259] HINT: Future log output will appear in directory "log". done server started + test -f standby1/postmaster.pid + echo 1 1 + rm -rf standby1 + pg_basebackup -D standby1 -p 5000 -X fetch --verbose --write-recovery-conf pg_basebackup: initiating base backup, waiting for checkpoint to complete pg_basebackup: checkpoint completed pg_basebackup: write-ahead log start point: 0/800028 on timeline 1 pg_basebackup: write-ahead log end point: 0/800100 pg_basebackup: syncing data to disk ... pg_basebackup: renaming backup_manifest.tmp to backup_manifest pg_basebackup: base backup completed + pg_ctl -w -D standby1 start -o '-p 5001' waiting for server to start....2024-08-08 14:36:06.573 +05 [2272] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:06.573 +05 [2272] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:06.576 +05 [2272] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:06.576 +05 [2272] DEBUG: created dynamic shared memory control segment 132029364 (26976 bytes) 2024-08-08 14:36:06.576 +05 [2272] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:06.576 +05 [2272] LOG: redirecting log output to logging collector process 2024-08-08 14:36:06.576 +05 [2272] HINT: Future log output will appear in directory "log". done server started + test -f standby2/postmaster.pid + echo 1 1 + rm -rf standby2 + pg_basebackup -D standby2 -p 5000 -X fetch --verbose --write-recovery-conf pg_basebackup: initiating base backup, waiting for checkpoint to complete pg_basebackup: checkpoint completed pg_basebackup: write-ahead log start point: 0/900028 on timeline 1 pg_basebackup: write-ahead log end point: 0/900100 pg_basebackup: syncing data to disk ... pg_basebackup: renaming backup_manifest.tmp to backup_manifest pg_basebackup: base backup completed + pg_ctl -w -D standby2 start -o '-p 5002' waiting for server to start....2024-08-08 14:36:07.412 +05 [2284] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:07.412 +05 [2284] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:07.415 +05 [2284] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:07.415 +05 [2284] DEBUG: created dynamic shared memory control segment 1345573086 (26976 bytes) 2024-08-08 14:36:07.415 +05 [2284] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:07.415 +05 [2284] LOG: redirecting log output to logging collector process 2024-08-08 14:36:07.415 +05 [2284] HINT: Future log output will appear in directory "log". done server started + psql -d postgres -a --no-psqlrc -p 5000 create table a (a int, b int); CREATE TABLE insert into a select i, i from generate_series(1, 100000) i; INSERT 0 100000 + pg_ctl -w -D master stop -m fast waiting for server to shut down....2024-08-08 14:36:07.933 +05 [2260] DEBUG: logger shutting down done server stopped + pg_ctl -w -D standby1 stop -m fast waiting for server to shut down....2024-08-08 14:36:08.011 +05 [2273] DEBUG: logger shutting down done server stopped + pg_ctl -w -D standby2 stop -m fast waiting for server to shut down....2024-08-08 14:36:08.111 +05 [2285] DEBUG: logger shutting down done server stopped + pg_ctl -w -D master start -o '-p 5000' waiting for server to start....2024-08-08 14:36:08.154 +05 [2300] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:08.154 +05 [2300] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:08.157 +05 [2300] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:08.157 +05 [2300] DEBUG: created dynamic shared memory control segment 1848579776 (26976 bytes) 2024-08-08 14:36:08.157 +05 [2300] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:08.157 +05 [2300] LOG: redirecting log output to logging collector process 2024-08-08 14:36:08.157 +05 [2300] HINT: Future log output will appear in directory "log". done server started + pg_ctl -w -D standby1 start -o '-p 5001' waiting for server to start....2024-08-08 14:36:08.259 +05 [2310] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:08.259 +05 [2310] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:08.262 +05 [2310] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:08.263 +05 [2310] DEBUG: created dynamic shared memory control segment 2087008324 (26976 bytes) 2024-08-08 14:36:08.263 +05 [2310] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:08.263 +05 [2310] LOG: redirecting log output to logging collector process 2024-08-08 14:36:08.263 +05 [2310] HINT: Future log output will appear in directory "log". done server started + pg_ctl -w -D standby2 start -o '-p 5002' waiting for server to start....2024-08-08 14:36:08.366 +05 [2319] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:08.366 +05 [2319] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:08.369 +05 [2319] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:08.369 +05 [2319] DEBUG: created dynamic shared memory control segment 1104889730 (26976 bytes) 2024-08-08 14:36:08.369 +05 [2319] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:08.369 +05 [2319] LOG: redirecting log output to logging collector process 2024-08-08 14:36:08.369 +05 [2319] HINT: Future log output will appear in directory "log". done server started + pg_controldata master + grep 'Latest checkpoint'"'"'s REDO WAL file:' + cut -d : -f 2 + grep -oP '\w+' + LAST_MASTER_WAL_FILE=000000010000000000000010 + pg_ctl -w -D standby1 promote waiting for server to promote.... done server promoted + pg_isready -p 5001 /tmp:5001 - accepting connections + pg_controldata standby1 + grep 'Latest checkpoint'"'"'s REDO WAL file:' + cut -d : -f 2 + grep -oP '\w+' + LAST_STANDBY_WAL_FILE=000000020000000000000010 + find standby1/pg_wal -name '*.history' -type f -print0 + xargs -r -0 ls -1 -t + head -1 + cat standby1/pg_wal/00000002.history 1 0/10392E8 no recovery target specified + pg_controldata master + grep checkpoint Latest checkpoint location: 0/1039270 Latest checkpoint's REDO location: 0/1039270 Latest checkpoint's REDO WAL file: 000000010000000000000010 Latest checkpoint's TimeLineID: 1 Latest checkpoint's PrevTimeLineID: 1 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:732 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 0 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:07 2024 + pg_controldata standby1 + grep checkpoint Latest checkpoint location: 0/1039350 Latest checkpoint's REDO location: 0/1039318 Latest checkpoint's REDO WAL file: 000000020000000000000010 Latest checkpoint's TimeLineID: 2 Latest checkpoint's PrevTimeLineID: 2 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:732 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 732 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:08 2024 + pg_controldata standby2 + grep checkpoint Latest checkpoint location: 0/1039270 Latest checkpoint's REDO location: 0/1039270 Latest checkpoint's REDO WAL file: 000000010000000000000010 Latest checkpoint's TimeLineID: 1 Latest checkpoint's PrevTimeLineID: 1 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:732 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 0 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:07 2024 + pg_waldump -p master 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1039270: invalid record length at 0/10392E8: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039048, prev 0/01039008, desc: INSERT off: 101, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039088, prev 0/01039048, desc: INSERT off: 102, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010390C8, prev 0/01039088, desc: INSERT off: 103, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039108, prev 0/010390C8, desc: INSERT off: 104, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039148, prev 0/01039108, desc: INSERT off: 105, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039188, prev 0/01039148, desc: INSERT off: 106, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + pg_waldump -p standby1 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1039270: invalid record length at 0/10392E8: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039048, prev 0/01039008, desc: INSERT off: 101, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039088, prev 0/01039048, desc: INSERT off: 102, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010390C8, prev 0/01039088, desc: INSERT off: 103, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039108, prev 0/010390C8, desc: INSERT off: 104, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039148, prev 0/01039108, desc: INSERT off: 105, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039188, prev 0/01039148, desc: INSERT off: 106, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + pg_waldump -p standby1 000000020000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/10393C8: unexpected pageaddr 0/1038000 in WAL segment 000000020000000000000010, LSN 0/103A000, offset 237568 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039148, prev 0/01039108, desc: INSERT off: 105, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039188, prev 0/01039148, desc: INSERT off: 106, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown rmgr: XLOG len (rec/tot): 42/ 42, tx: 0, lsn: 0/010392E8, prev 0/01039270, desc: END_OF_RECOVERY tli 2; prev tli 1; time 2024-08-08 14:36:08.470837 +05 rmgr: Standby len (rec/tot): 50/ 50, tx: 0, lsn: 0/01039318, prev 0/010392E8, desc: RUNNING_XACTS nextXid 732 latestCompletedXid 731 oldestRunningXid 732 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039350, prev 0/01039318, desc: CHECKPOINT_ONLINE redo 0/1039318; tli 2; prev tli 2; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 732; online rmgr: XLOG len (rec/tot): 49/ 885, tx: 0, lsn: 0/010393C8, prev 0/01039350, desc: FPI_FOR_HINT , blkref #0: rel 1663/5/1259 blk 0 FPW + pg_waldump -p standby2 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1039270: invalid record length at 0/10392E8: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039048, prev 0/01039008, desc: INSERT off: 101, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039088, prev 0/01039048, desc: INSERT off: 102, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010390C8, prev 0/01039088, desc: INSERT off: 103, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039108, prev 0/010390C8, desc: INSERT off: 104, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039148, prev 0/01039108, desc: INSERT off: 105, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039188, prev 0/01039148, desc: INSERT off: 106, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + psql -d postgres -a --no-psqlrc -p 5000 --set synchronous_standby_names = 'postgres'; --select pg_reload_conf(); insert into a select i, i from generate_series(1, 340) i; INSERT 0 340 --commit; + pg_ctl -w -D standby2 stop -m fast waiting for server to shut down....2024-08-08 14:36:08.621 +05 [2320] DEBUG: logger shutting down done server stopped + pg_ctl -w -D master stop -m fast waiting for server to shut down....2024-08-08 14:36:08.728 +05 [2301] DEBUG: logger shutting down done server stopped + pg_ctl -w -D standby1 stop -m fast waiting for server to shut down....2024-08-08 14:36:08.827 +05 [2311] DEBUG: logger shutting down done server stopped + pg_controldata master + grep checkpoint Latest checkpoint location: 0/1041370 Latest checkpoint's REDO location: 0/1041370 Latest checkpoint's REDO WAL file: 000000010000000000000010 Latest checkpoint's TimeLineID: 1 Latest checkpoint's PrevTimeLineID: 1 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:733 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 0 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:08 2024 + pg_controldata standby1 + grep checkpoint Latest checkpoint location: 0/103B2F0 Latest checkpoint's REDO location: 0/103B2F0 Latest checkpoint's REDO WAL file: 000000020000000000000010 Latest checkpoint's TimeLineID: 2 Latest checkpoint's PrevTimeLineID: 2 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:732 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 0 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:08 2024 + pg_controldata standby2 + grep checkpoint Latest checkpoint location: 0/1039270 Latest checkpoint's REDO location: 0/1039270 Latest checkpoint's REDO WAL file: 000000010000000000000010 Latest checkpoint's TimeLineID: 1 Latest checkpoint's PrevTimeLineID: 1 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0:732 Latest checkpoint's NextOID: 16387 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 722 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 0 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTsXid:0 Latest checkpoint's newestCommitTsXid:0 Time of latest checkpoint: Thu Aug 8 14:36:07 2024 + pg_waldump -p master 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1041370: invalid record length at 0/10413E8: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041148, prev 0/01041108, desc: INSERT off: 215, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041188, prev 0/01041148, desc: INSERT off: 216, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/010411C8, prev 0/01041188, desc: INSERT off: 217, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041208, prev 0/010411C8, desc: INSERT off: 218, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041248, prev 0/01041208, desc: INSERT off: 219, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041288, prev 0/01041248, desc: INSERT off: 220, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/010412C8, prev 0/01041288, desc: INSERT off: 221, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041308, prev 0/010412C8, desc: INSERT off: 222, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Transaction len (rec/tot): 34/ 34, tx: 732, lsn: 0/01041348, prev 0/01041308, desc: COMMIT 2024-08-08 14:36:08.610124 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01041370, prev 0/01041348, desc: CHECKPOINT_SHUTDOWN redo 0/1041370; tli 1; prev tli 1; fpw true; xid 0:733; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + pg_waldump -p standby1 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1039270: invalid record length at 0/10392E8: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039048, prev 0/01039008, desc: INSERT off: 101, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039088, prev 0/01039048, desc: INSERT off: 102, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010390C8, prev 0/01039088, desc: INSERT off: 103, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039108, prev 0/010390C8, desc: INSERT off: 104, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039148, prev 0/01039108, desc: INSERT off: 105, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039188, prev 0/01039148, desc: INSERT off: 106, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + pg_waldump -p standby1 000000020000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/103B2F0: invalid record length at 0/103B368: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/010391C8, prev 0/01039188, desc: INSERT off: 107, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Heap len (rec/tot): 63/ 63, tx: 731, lsn: 0/01039208, prev 0/010391C8, desc: INSERT off: 108, flags: 0x00, blkref #0: rel 1663/5/16384 blk 442 rmgr: Transaction len (rec/tot): 34/ 34, tx: 731, lsn: 0/01039248, prev 0/01039208, desc: COMMIT 2024-08-08 14:36:07.722423 +05 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039270, prev 0/01039248, desc: CHECKPOINT_SHUTDOWN redo 0/1039270; tli 1; prev tli 1; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown rmgr: XLOG len (rec/tot): 42/ 42, tx: 0, lsn: 0/010392E8, prev 0/01039270, desc: END_OF_RECOVERY tli 2; prev tli 1; time 2024-08-08 14:36:08.470837 +05 rmgr: Standby len (rec/tot): 50/ 50, tx: 0, lsn: 0/01039318, prev 0/010392E8, desc: RUNNING_XACTS nextXid 732 latestCompletedXid 731 oldestRunningXid 732 rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01039350, prev 0/01039318, desc: CHECKPOINT_ONLINE redo 0/1039318; tli 2; prev tli 2; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 732; online rmgr: XLOG len (rec/tot): 49/ 885, tx: 0, lsn: 0/010393C8, prev 0/01039350, desc: FPI_FOR_HINT , blkref #0: rel 1663/5/1259 blk 0 FPW rmgr: XLOG len (rec/tot): 49/ 7061, tx: 0, lsn: 0/01039740, prev 0/010393C8, desc: FPI_FOR_HINT , blkref #0: rel 1663/5/1249 blk 17 FPW rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/0103B2F0, prev 0/01039740, desc: CHECKPOINT_SHUTDOWN redo 0/103B2F0; tli 2; prev tli 2; fpw true; xid 0:732; oid 16387; multi 1; offset 0; oldest xid 722 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0; oldest running xid 0; shutdown + pg_waldump -p standby2 000000010000000000000010 + tail -n 10 pg_waldump: error: error in WAL record at 0/1041348: invalid record length at 0/1041370: expected at least 24, got 0 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041108, prev 0/010410C8, desc: INSERT off: 214, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041148, prev 0/01041108, desc: INSERT off: 215, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041188, prev 0/01041148, desc: INSERT off: 216, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/010411C8, prev 0/01041188, desc: INSERT off: 217, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041208, prev 0/010411C8, desc: INSERT off: 218, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041248, prev 0/01041208, desc: INSERT off: 219, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041288, prev 0/01041248, desc: INSERT off: 220, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/010412C8, prev 0/01041288, desc: INSERT off: 221, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Heap len (rec/tot): 63/ 63, tx: 732, lsn: 0/01041308, prev 0/010412C8, desc: INSERT off: 222, flags: 0x00, blkref #0: rel 1663/5/16384 blk 443 rmgr: Transaction len (rec/tot): 34/ 34, tx: 732, lsn: 0/01041348, prev 0/01041308, desc: COMMIT 2024-08-08 14:36:08.610124 +05 + pg_rewind --progress --debug '--source-pgdata=standby1' '--target-pgdata=standby2' pg_rewind: Source timeline history: pg_rewind: Target timeline history: pg_rewind: 1: 0/0 - 0/0 pg_rewind: servers diverged at WAL location 0/10392E8 on timeline 1 pg_rewind: no rewind required + cat + touch standby2/standby.signal + pg_ctl -w -D standby1 start -o '-p 5001' waiting for server to start....2024-08-08 14:36:08.957 +05 [2386] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:08.957 +05 [2386] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:08.961 +05 [2386] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:08.961 +05 [2386] DEBUG: created dynamic shared memory control segment 2166747262 (26976 bytes) 2024-08-08 14:36:08.961 +05 [2386] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:08.961 +05 [2386] LOG: redirecting log output to logging collector process 2024-08-08 14:36:08.961 +05 [2386] HINT: Future log output will appear in directory "log". done server started + pg_ctl -w -D standby2 start -o '-p 5002' waiting for server to start....2024-08-08 14:36:09.063 +05 [2396] DEBUG: registering background worker "logical replication launcher" 2024-08-08 14:36:09.063 +05 [2396] DEBUG: mmap(146800640) with MAP_HUGETLB failed, huge pages disabled: Out of memory 2024-08-08 14:36:09.067 +05 [2396] DEBUG: dynamic shared memory system will support 674 segments 2024-08-08 14:36:09.067 +05 [2396] DEBUG: created dynamic shared memory control segment 373333278 (26976 bytes) 2024-08-08 14:36:09.067 +05 [2396] DEBUG: max_safe_fds = 986, usable_fds = 1000, already_open = 4 2024-08-08 14:36:09.067 +05 [2396] LOG: redirecting log output to logging collector process 2024-08-08 14:36:09.067 +05 [2396] HINT: Future log output will appear in directory "log". done server started + psql -d postgres -a --no-psqlrc -p 5001 select count(*) from a; count -------- 100000 (1 row) + psql -d postgres -a --no-psqlrc -p 5002 select count(*) from a; count -------- 100340 (1 row) + pg_ctl -w -D standby1 stop -m fast waiting for server to shut down....2024-08-08 14:36:09.316 +05 [2387] DEBUG: logger shutting down done server stopped + pg_ctl -w -D standby2 stop -m fast waiting for server to shut down....2024-08-08 14:36:09.355 +05 [2397] DEBUG: logger shutting down done server stopped