From b1a7df1ab29de31cbbe51ae993d3368eb8ecbfec Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 18 Nov 2021 16:31:22 -0800 Subject: [PATCH v1] wip: test heap_prune_page() bug recently dead -> dead inbetween repeated accesses. --- .../expected/prune-recently-dead.out | 241 ++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/prune-recently-dead.spec | 229 +++++++++++++++++ 3 files changed, 471 insertions(+) create mode 100644 src/test/isolation/expected/prune-recently-dead.out create mode 100644 src/test/isolation/specs/prune-recently-dead.spec diff --git a/src/test/isolation/expected/prune-recently-dead.out b/src/test/isolation/expected/prune-recently-dead.out new file mode 100644 index 00000000000..a79bdb92f9b --- /dev/null +++ b/src/test/isolation/expected/prune-recently-dead.out @@ -0,0 +1,241 @@ +Parsed test spec with 9 sessions + +starting permutation: mon_vacuum mon_slot s1_insert_1 s1_update_1 s1_update_1 vac_vacuum s1_insert_2 mon_page s2_begin_rr s2_xid s3_begin_rr s3_snap s1_delete_2 s4_begin_rr s4_xid s1_delete_1 s7_begin_rc s7_block_idx_1 s8_begin_rc s8_block_idx_2 s9_begin_rc s9_block_idx_3 vac_vacuum mon_state mon_page mon_locks s2_commit s7_inval_idx_1 s7_commit s3_commit s8_rollback s4_commit mon_page mon_state mon_locks s9_rollback mon_page mon_view mon_locks mon_verify s1_insert_17 mon_page s1_select_1 mon_verify +step mon_vacuum: VACUUM pg_class; +step mon_slot: SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'test_decoding'); +?column? +-------- +init +(1 row) + +step s1_insert_1: + INSERT INTO many_updates (id) VALUES(1); + +step s1_update_1: + UPDATE many_updates SET counter = counter + 1 WHERE id = 1 RETURNING * /*, ctid, xmin, */; + +id|counter|indexed +--+-------+------- + 1| 1| 0 +(1 row) + +step s1_update_1: + UPDATE many_updates SET counter = counter + 1 WHERE id = 1 RETURNING * /*, ctid, xmin, */; + +id|counter|indexed +--+-------+------- + 1| 2| 0 +(1 row) + +step vac_vacuum: VACUUM (skip_locked, index_cleanup off, truncate off, verbose off) many_updates; +step s1_insert_2: + INSERT INTO many_updates (id) VALUES(2); + +step mon_page: SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask +lp|lp_off|lp_flags|lp_len|t_field3|t_ctid|t_infomask2|t_infomask|raw_flags |combined_flags|t_hoff|t_bits +--+------+--------+------+--------+------+-----------+----------+--------------------------------------------------------------------+--------------+------+------ + 1| 3| 2| 0| | | | | | | | + 2| 8112| 1| 36| 0|(0,2) | 3| 2048|{HEAP_XMAX_INVALID} |{} | 24| + 3| 8152| 1| 36| 0|(0,3) | 32771| 10496|{HEAP_XMIN_COMMITTED,HEAP_XMAX_INVALID,HEAP_UPDATED,HEAP_ONLY_TUPLE}|{} | 24| +(3 rows) + +step s2_begin_rr: BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; +step s2_xid: SELECT txid_current() <> '0'; +?column? +-------- +t +(1 row) + +step s3_begin_rr: BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; +step s3_snap: SELECT 'just for a snapshot s3'; +?column? +---------------------- +just for a snapshot s3 +(1 row) + +step s1_delete_2: + DELETE FROM many_updates WHERE id = 2 RETURNING * /*, ctid, xmin, */; + +id|counter|indexed +--+-------+------- + 2| 0| 0 +(1 row) + +step s4_begin_rr: BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; +step s4_xid: SELECT txid_current() <> '0'; +?column? +-------- +t +(1 row) + +step s1_delete_1: + DELETE FROM many_updates WHERE id = 1 RETURNING * /*, ctid, xmin, */; + +id|counter|indexed +--+-------+------- + 1| 2| 0 +(1 row) + +step s7_begin_rc: BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; +step s7_block_idx_1: ALTER INDEX many_updates_block_1 SET TABLESPACE pg_default; +step s8_begin_rc: BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; +step s8_block_idx_2: ALTER INDEX many_updates_block_2 SET TABLESPACE pg_default; +step s9_begin_rc: BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; +step s9_block_idx_3: ALTER INDEX many_updates_block_3 SET TABLESPACE pg_default; +step vac_vacuum: VACUUM (skip_locked, index_cleanup off, truncate off, verbose off) many_updates; +step mon_state: SELECT /* pid,*/ /* backend_xid, backend_xmin, */ application_name, query FROM pg_stat_activity WHERE application_name LIKE 'isolation/prune-recently-dead/%'; +application_name |query +---------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------- +isolation/prune-recently-dead/s1 | + DELETE FROM many_updates WHERE id = 1 RETURNING * /*, ctid, xmin, */; + +isolation/prune-recently-dead/s2 |SELECT txid_current() <> '0'; +isolation/prune-recently-dead/s3 |SELECT 'just for a snapshot s3'; +isolation/prune-recently-dead/s4 |SELECT txid_current() <> '0'; +isolation/prune-recently-dead/s7 |ALTER INDEX many_updates_block_1 SET TABLESPACE pg_default; +isolation/prune-recently-dead/s8 |ALTER INDEX many_updates_block_2 SET TABLESPACE pg_default; +isolation/prune-recently-dead/s9 |ALTER INDEX many_updates_block_3 SET TABLESPACE pg_default; +isolation/prune-recently-dead/vac|VACUUM (skip_locked, index_cleanup off, truncate off, verbose off) many_updates; +isolation/prune-recently-dead/mon|SELECT /* pid,*/ /* backend_xid, backend_xmin, */ application_name, query FROM pg_stat_activity WHERE application_name LIKE 'isolation/prune-recently-dead/%'; +(9 rows) + +step mon_page: SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask +lp|lp_off|lp_flags|lp_len|t_field3|t_ctid|t_infomask2|t_infomask|raw_flags |combined_flags|t_hoff|t_bits +--+------+--------+------+--------+------+-----------+----------+--------------------------------------------------------------------+--------------+------+------ + 1| 3| 2| 0| | | | | | | | + 2| 8112| 1| 36| 0|(0,2) | 8195| 1280|{HEAP_XMIN_COMMITTED,HEAP_XMAX_COMMITTED,HEAP_KEYS_UPDATED} |{} | 24| + 3| 8152| 1| 36| 0|(0,3) | 40963| 8448|{HEAP_XMIN_COMMITTED,HEAP_UPDATED,HEAP_KEYS_UPDATED,HEAP_ONLY_TUPLE}|{} | 24| +(3 rows) + +step mon_locks: SELECT /* pid, */ granted, mode, locktype, relation::regclass FROM pg_locks WHERE database = (SELECT oid FROM pg_database WHERE datname = current_database()) ORDER BY locktype, mode, relation::regclass, granted; +granted|mode |locktype|relation +-------+------------------------+--------+------------------------ +t |AccessExclusiveLock |relation|many_updates_block_1 +t |AccessExclusiveLock |relation|many_updates_block_2 +t |AccessExclusiveLock |relation|many_updates_block_3 +t |AccessShareLock |relation|pg_locks +t |RowExclusiveLock |relation|many_updates_id_idx +t |RowExclusiveLock |relation|many_updates_indexed_idx +f |RowExclusiveLock |relation|many_updates_block_1 +t |ShareUpdateExclusiveLock|relation|many_updates +(8 rows) + +step s2_commit: COMMIT; +step s7_inval_idx_1: ALTER INDEX many_updates_block_1 SET(FILLFACTOR = 90); +step s7_commit: COMMIT; +step s3_commit: COMMIT; +step s8_rollback: ROLLBACK; +step s4_commit: COMMIT; +step mon_page: SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask +lp|lp_off|lp_flags|lp_len|t_field3|t_ctid|t_infomask2|t_infomask|raw_flags |combined_flags|t_hoff|t_bits +--+------+--------+------+--------+------+-----------+----------+--------------------------------------------------------------------+--------------+------+------ + 1| 3| 2| 0| | | | | | | | + 2| 8112| 1| 36| 0|(0,2) | 8195| 1280|{HEAP_XMIN_COMMITTED,HEAP_XMAX_COMMITTED,HEAP_KEYS_UPDATED} |{} | 24| + 3| 8152| 1| 36| 0|(0,3) | 40963| 8448|{HEAP_XMIN_COMMITTED,HEAP_UPDATED,HEAP_KEYS_UPDATED,HEAP_ONLY_TUPLE}|{} | 24| +(3 rows) + +step mon_state: SELECT /* pid,*/ /* backend_xid, backend_xmin, */ application_name, query FROM pg_stat_activity WHERE application_name LIKE 'isolation/prune-recently-dead/%'; +application_name |query +---------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------- +isolation/prune-recently-dead/s1 | + DELETE FROM many_updates WHERE id = 1 RETURNING * /*, ctid, xmin, */; + +isolation/prune-recently-dead/s2 |COMMIT; +isolation/prune-recently-dead/s3 |COMMIT; +isolation/prune-recently-dead/s4 |COMMIT; +isolation/prune-recently-dead/s7 |COMMIT; +isolation/prune-recently-dead/s8 |ROLLBACK; +isolation/prune-recently-dead/s9 |ALTER INDEX many_updates_block_3 SET TABLESPACE pg_default; +isolation/prune-recently-dead/vac|VACUUM (skip_locked, index_cleanup off, truncate off, verbose off) many_updates; +isolation/prune-recently-dead/mon|SELECT /* pid,*/ /* backend_xid, backend_xmin, */ application_name, query FROM pg_stat_activity WHERE application_name LIKE 'isolation/prune-recently-dead/%'; +(9 rows) + +step mon_locks: SELECT /* pid, */ granted, mode, locktype, relation::regclass FROM pg_locks WHERE database = (SELECT oid FROM pg_database WHERE datname = current_database()) ORDER BY locktype, mode, relation::regclass, granted; +granted|mode |locktype|relation +-------+------------------------+--------+------------------------ +t |AccessExclusiveLock |relation|many_updates_block_3 +t |AccessShareLock |relation|pg_locks +t |RowExclusiveLock |relation|many_updates_id_idx +t |RowExclusiveLock |relation|many_updates_indexed_idx +t |RowExclusiveLock |relation|many_updates_block_1 +t |RowExclusiveLock |relation|many_updates_block_2 +f |RowExclusiveLock |relation|many_updates_block_3 +t |ShareUpdateExclusiveLock|relation|many_updates +(8 rows) + +step s9_rollback: ROLLBACK; +step vac_vacuum: <... completed> +step mon_page: SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask +lp|lp_off|lp_flags|lp_len|t_field3|t_ctid|t_infomask2|t_infomask|raw_flags|combined_flags|t_hoff|t_bits +--+------+--------+------+--------+------+-----------+----------+---------+--------------+------+------ + 1| 3| 2| 0| | | | | | | | + 2| 0| 3| 0| | | | | | | | + 3| 0| 0| 0| | | | | | | | +(3 rows) + +step mon_view: + SELECT * FROM many_updates; + +id|counter|indexed +--+-------+------- +(0 rows) + +step mon_locks: SELECT /* pid, */ granted, mode, locktype, relation::regclass FROM pg_locks WHERE database = (SELECT oid FROM pg_database WHERE datname = current_database()) ORDER BY locktype, mode, relation::regclass, granted; +granted|mode |locktype|relation +-------+---------------+--------+-------- +t |AccessShareLock|relation|pg_locks +(1 row) + +step mon_verify: + SELECT * FROM verify_heapam('many_updates'); + SELECT * FROM bt_index_parent_check('many_updates_id_idx', true, true); + +blkno|offnum|attnum|msg +-----+------+------+--------------------------------------------------- + 0| 1| |line pointer redirection to unused item at offset 3 +(1 row) + +bt_index_parent_check +--------------------- + +(1 row) + +step s1_insert_17: + INSERT INTO many_updates (id) VALUES(17); + +step mon_page: SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask +lp|lp_off|lp_flags|lp_len|t_field3|t_ctid|t_infomask2|t_infomask|raw_flags |combined_flags|t_hoff|t_bits +--+------+--------+------+--------+------+-----------+----------+-------------------+--------------+------+------ + 1| 3| 2| 0| | | | | | | | + 2| 0| 3| 0| | | | | | | | + 3| 8152| 1| 36| 0|(0,3) | 3| 2048|{HEAP_XMAX_INVALID}|{} | 24| +(3 rows) + +step s1_select_1: + SET LOCAL enable_seqscan = false; + SELECT ctid, /*xmin, xmax, */ id FROM many_updates WHERE id = 1; + RESET enable_seqscan; + +ctid |id +-----+-- +(0,3)|17 +(1 row) + +step mon_verify: + SELECT * FROM verify_heapam('many_updates'); + SELECT * FROM bt_index_parent_check('many_updates_id_idx', true, true); + +blkno|offnum|attnum|msg +-----+------+------+--- +(0 rows) + +bt_index_parent_check +--------------------- + +(1 row) + +pg_drop_replication_slot +------------------------ + +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index f4c01006fc1..06accc21932 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -1,3 +1,4 @@ +test: prune-recently-dead test: read-only-anomaly test: read-only-anomaly-2 test: read-only-anomaly-3 diff --git a/src/test/isolation/specs/prune-recently-dead.spec b/src/test/isolation/specs/prune-recently-dead.spec new file mode 100644 index 00000000000..97edd7fbc4a --- /dev/null +++ b/src/test/isolation/specs/prune-recently-dead.spec @@ -0,0 +1,229 @@ +setup +{ + DROP TABLE IF EXISTS many_updates; + DROP EXTENSION IF EXISTS pageinspect; + DROP EXTENSION IF EXISTS amcheck; + + CREATE EXTENSION pageinspect; + CREATE EXTENSION amcheck; + + CREATE TABLE many_updates(id int not null, counter int default 0, indexed int default 0); + CREATE INDEX ON many_updates(id); + CREATE INDEX ON many_updates(indexed); + CREATE INDEX many_updates_block_1 ON many_updates(indexed); + CREATE INDEX many_updates_block_2 ON many_updates(indexed); + CREATE INDEX many_updates_block_3 ON many_updates(indexed); +} + +teardown +{ + --DROP TABLE many_updates; + + --DROP EXTENSION pageinspect; + SELECT pg_drop_replication_slot('test_slot'); +} + +session s1 +setup {SET application_name = 'isolation/prune-recently-dead/s1'} + +step s1_insert_1 { + INSERT INTO many_updates (id) VALUES(1); +} + +step s1_insert_2 { + INSERT INTO many_updates (id) VALUES(2); +} + +step s1_insert_17 { + INSERT INTO many_updates (id) VALUES(17); +} + +step s1_update_1 { + UPDATE many_updates SET counter = counter + 1 WHERE id = 1 RETURNING * /*, ctid, xmin, */; +} + +step s1_delete_1 { + DELETE FROM many_updates WHERE id = 1 RETURNING * /*, ctid, xmin, */; +} + +step s1_delete_2 { + DELETE FROM many_updates WHERE id = 2 RETURNING * /*, ctid, xmin, */; +} + +step s1_select_1 { + SET LOCAL enable_seqscan = false; + SELECT ctid, /*xmin, xmax, */ id FROM many_updates WHERE id = 1; + RESET enable_seqscan; +} + +session s2 +setup {SET application_name = 'isolation/prune-recently-dead/s2'} + +step s2_begin_rr { BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; } +step s2_xid { SELECT txid_current() <> '0'; } +step s2_commit { COMMIT; } + +session s3 +setup {SET application_name = 'isolation/prune-recently-dead/s3'} + +step s3_begin_rr { BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; } +step s3_snap { SELECT 'just for a snapshot s3'; } +step s3_commit { COMMIT; } + +session s4 +setup {SET application_name = 'isolation/prune-recently-dead/s4'} + +step s4_begin_rr { BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; } +step s4_xid { SELECT txid_current() <> '0'; } +step s4_commit { COMMIT; } + + + +session s7 +setup {SET application_name = 'isolation/prune-recently-dead/s7'} + +step s7_begin_rc { BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; } +step s7_block_idx_1 { ALTER INDEX many_updates_block_1 SET TABLESPACE pg_default; } +step s7_inval_idx_1 { ALTER INDEX many_updates_block_1 SET(FILLFACTOR = 90); } + +step s7_commit { COMMIT; } + + +session s8 +setup {SET application_name = 'isolation/prune-recently-dead/s8'} + +step s8_begin_rc { BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; } +step s8_block_idx_2 { ALTER INDEX many_updates_block_2 SET TABLESPACE pg_default; } +step s8_rollback { ROLLBACK; } + + +session s9 +setup {SET application_name = 'isolation/prune-recently-dead/s9'} + +step s9_begin_rc { BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; } +step s9_block_idx_3 { ALTER INDEX many_updates_block_3 SET TABLESPACE pg_default; } +step s9_rollback { ROLLBACK; } + + + +session vac +setup {SET application_name = 'isolation/prune-recently-dead/vac'} + +#step vac_relcache {SELECT count(*) FROM many_updates;} +step vac_vacuum {VACUUM (skip_locked, index_cleanup off, truncate off, verbose off) many_updates;} + +session mon +setup {SET application_name = 'isolation/prune-recently-dead/mon'} + +step mon_slot { SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');} + +step mon_state {SELECT /* pid,*/ /* backend_xid, backend_xmin, */ application_name, query FROM pg_stat_activity WHERE application_name LIKE 'isolation/prune-recently-dead/%';} + +step mon_page {SELECT lp, lp_off, lp_flags, lp_len, /* t_xmin, t_xmax, */ t_field3, t_ctid, t_infomask2, t_infomask, mask.raw_flags, mask.combined_flags, t_hoff, t_bits FROM heap_page_items(get_raw_page('many_updates', 0)), heap_tuple_infomask_flags(t_infomask, t_infomask2) AS mask} + +step mon_locks {SELECT /* pid, */ granted, mode, locktype, relation::regclass FROM pg_locks WHERE database = (SELECT oid FROM pg_database WHERE datname = current_database()) ORDER BY locktype, mode, relation::regclass, granted; } + +step mon_view { + SELECT * FROM many_updates; +} + +step mon_vacuum {VACUUM pg_class;} + +step mon_verify { + SELECT * FROM verify_heapam('many_updates'); + SELECT * FROM bt_index_parent_check('many_updates_id_idx', true, true); +} + +permutation + + # avoid hot pruning on catalogs + mon_vacuum + mon_slot + + s1_insert_1 # lp1 + s1_update_1 # lp1 -> lp2 + s1_update_1 # lp2 -> lp3 + vac_vacuum # redirect lp1 -> lp3, mark lp2 unused + s1_insert_2 # lp2 + + mon_page + + # hold xmin to one before deletion of lp2, so that RecentXmin changes + s2_begin_rr + s2_xid + + # hold xmin before deletion of lp2 + s3_begin_rr + s3_snap + + # delete lp2 + s1_delete_2 + + # hold xmin before update of lp3 + s4_begin_rr + s4_xid + + s1_delete_1 + + # lock indexes, this will let us do stuff after vacuum_set_xid_limits + s7_begin_rc + s7_block_idx_1 + + s8_begin_rc + s8_block_idx_2 + + s9_begin_rc + s9_block_idx_3 + + + # finally start vacuuming, will block when locking + # the first index + vac_vacuum + + mon_state + mon_page + mon_locks + + # increase RecentXmin + s2_commit + + # Trigger invalidations to cause vac_open_indexes to build a snapshot. + s7_inval_idx_1 + + # and release, this will process invalidations, and then block on the next index + s7_commit + + # Increase horizon enough that lp2 can be deleted after a recheck + s3_commit + + # Release 2nd blocking index + s8_rollback + + # Now we can allow vacuuming + s4_commit + + mon_page + mon_state + mon_locks + + # Release 3nd blocking index + s9_rollback + + mon_page + mon_view + mon_locks + + # the bug will now have caused lp3 to be freed, while lp1 still points to lp3 + + # amcheck can complain about a redirection to unused lp + mon_verify + + # insert a different index tuple into the now wrongly freed slot + s1_insert_17 + mon_page + + # and the new row is visible via the wrong index entry + s1_select_1 + + # and amcheck now doesn't notice a problem anymore + mon_verify -- 2.34.0