From 22c0254b76be05a2235ed84268d189fa6785e844 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 5 Apr 2019 22:27:14 +0200 Subject: [PATCH v2] Report progress of REINDEX operations This uses the same infrastructure that the CREATE INDEX progress reporting uses. --- doc/src/sgml/monitoring.sgml | 52 +++++++++++++++++++++------- src/backend/catalog/index.c | 10 ++++++ src/backend/catalog/system_views.sql | 11 +++--- src/backend/commands/indexcmds.c | 39 ++++++++++++++++++--- src/include/commands/progress.h | 3 ++ src/test/regress/expected/rules.out | 11 +++--- 6 files changed, 102 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index b946e13fdc..4eb43f2de9 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -346,7 +346,7 @@ Dynamic Statistics Views pg_stat_progress_create_indexpg_stat_progress_create_index - One row for each backend running CREATE INDEX, showing + One row for each backend running CREATE INDEX or REINDEX, showing current progress. See . @@ -3477,7 +3477,7 @@ Progress Reporting CREATE INDEX Progress Reporting - Whenever CREATE INDEX is running, the + Whenever CREATE INDEX or REINDEX is running, the pg_stat_progress_create_index view will contain one row for each backend that is currently creating indexes. The tables below describe the information that will be reported and provide information @@ -3516,6 +3516,12 @@ <structname>pg_stat_progress_create_index</structname> View oid OID of the table on which the index is being created. + + index_relid + oid + OID of the index being created or reindexed. During a + non-concurrent CREATE INDEX, this is 0. + phase text @@ -3605,15 +3611,15 @@ CREATE INDEX phases initializing - CREATE INDEX is preparing to create the index. This + CREATE INDEX or REINDEX is preparing to create the index. This phase is expected to be very brief. - waiting for old snapshots + waiting for writers before build - CREATE INDEX CONCURRENTLY is waiting for transactions - that can potentially see the table to release their snapshots. + CREATE INDEX CONCURRENTLY or REINDEX CONCURRENTLY is waiting for transactions + with write locks that can potentially see the table to finish. This phase is skipped when not in concurrent mode. Columns lockers_total, lockers_done and current_locker_pid contain the progress @@ -3632,10 +3638,10 @@ CREATE INDEX phases - waiting for writer snapshots + waiting for writers before validation - CREATE INDEX CONCURRENTLY is waiting for transactions - that can potentially write into the table to release their snapshots. + CREATE INDEX CONCURRENTLY or REINDEX CONCURRENTLY is waiting for transactions + with write locks that can potentially write into the table to finish. This phase is skipped when not in concurrent mode. Columns lockers_total, lockers_done and current_locker_pid contain the progress @@ -3670,9 +3676,9 @@ CREATE INDEX phases - waiting for reader snapshots + waiting for old snapshots - CREATE INDEX CONCURRENTLY is waiting for transactions + CREATE INDEX CONCURRENTLY or REINDEX CONCURRENTLY is waiting for transactions that can potentially see the table to release their snapshots. This phase is skipped when not in concurrent mode. Columns lockers_total, lockers_done @@ -3680,6 +3686,28 @@ CREATE INDEX phases information for this phase. + + waiting for readers before marking dead + + REINDEX CONCURRENTLY is waiting for transactions + with read locks on the table to finish, before marking the old index dead. + This phase is skipped when not in concurrent mode. + Columns lockers_total, lockers_done + and current_locker_pid contain the progress + information for this phase. + + + + waiting for readers before dropping + + REINDEX CONCURRENTLY is waiting for transactions + with read locks on the table to finish, before dropping the old index. + This phase is skipped when not in concurrent mode. + Columns lockers_total, lockers_done + and current_locker_pid contain the progress + information for this phase. + + @@ -3937,7 +3965,7 @@ <structname>pg_stat_progress_cluster</structname> View cluster_index_relid - bigint + oid If the table is being scanned using an index, this is the OID of the index being used; otherwise, it is zero. diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 2ed7fdb021..9b1d546791 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3286,12 +3286,20 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, heapId = IndexGetRelation(indexId, false); heapRelation = table_open(heapId, ShareLock); + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, + heapId); + pgstat_progress_update_param(PROGRESS_CREATEIDX_INDEX_OID, + indexId); + /* * Open the target index relation and get an exclusive lock on it, to * ensure that no one else is touching this particular index. */ iRel = index_open(indexId, AccessExclusiveLock); + pgstat_progress_update_param(PROGRESS_CREATEIDX_ACCESS_METHOD_OID, + iRel->rd_rel->relam); + /* * The case of reindexing partitioned tables and indexes is handled * differently by upper layers, so this case shouldn't arise. @@ -3442,6 +3450,8 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, errdetail_internal("%s", pg_rusage_show(&ru0)))); + pgstat_progress_end_command(); + /* Close rels, but keep locks */ index_close(iRel, NoLock); table_close(heapRelation, NoLock); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 72f786d6f8..16e456a7d9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -933,7 +933,7 @@ CREATE VIEW pg_stat_progress_cluster AS WHEN 6 THEN 'rebuilding index' WHEN 7 THEN 'performing final cleanup' END AS phase, - S.param3 AS cluster_index_relid, + CAST(S.param3 AS oid) AS cluster_index_relid, S.param4 AS heap_tuples_scanned, S.param5 AS heap_tuples_written, S.param6 AS heap_blks_total, @@ -946,16 +946,19 @@ CREATE VIEW pg_stat_progress_create_index AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, S.relid AS relid, + CAST(S.param7 AS oid) AS index_relid, CASE S.param10 WHEN 0 THEN 'initializing' - WHEN 1 THEN 'waiting for old snapshots' + WHEN 1 THEN 'waiting for writers before build' WHEN 2 THEN 'building index' || COALESCE((': ' || pg_indexam_progress_phasename(S.param9::oid, S.param11)), '') - WHEN 3 THEN 'waiting for writer snapshots' + WHEN 3 THEN 'waiting for writers before validation' WHEN 4 THEN 'index validation: scanning index' WHEN 5 THEN 'index validation: sorting tuples' WHEN 6 THEN 'index validation: scanning table' - WHEN 7 THEN 'waiting for reader snapshots' + WHEN 7 THEN 'waiting for old snapshots' + WHEN 8 THEN 'waiting for readers before marking dead' + WHEN 9 THEN 'waiting for readers before dropping' END as phase, S.param4 AS lockers_total, S.param5 AS lockers_done, diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 348d543297..4a9b030c2c 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -478,6 +478,12 @@ DefineIndex(Oid relationId, pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, relationId); + /* + * No index OID to report yet + */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_INDEX_OID, + InvalidOid); + /* * count key attributes in index */ @@ -1244,6 +1250,12 @@ DefineIndex(Oid relationId, CommitTransactionCommand(); StartTransactionCommand(); + /* + * The index is now visible, so we can report the OID. + */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_INDEX_OID, + indexRelationId); + /* * Phase 2 of concurrent index build (see comments for validate_index() * for an overview of how this works) @@ -2873,6 +2885,13 @@ ReindexRelationConcurrently(Oid relationOid, int options) heapRel = table_open(indexRel->rd_index->indrelid, ShareUpdateExclusiveLock); + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, + RelationGetRelid(heapRel)); + pgstat_progress_update_param(PROGRESS_CREATEIDX_INDEX_OID, + indexId); + pgstat_progress_update_param(PROGRESS_CREATEIDX_ACCESS_METHOD_OID, + indexRel->rd_rel->relam); + /* Choose a temporary relation name for the new index */ concurrentName = ChooseRelationName(get_rel_name(indexId), NULL, @@ -2967,7 +2986,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) * DefineIndex() for more details. */ - WaitForLockersMultiple(lockTags, ShareLock, false); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_1); + WaitForLockersMultiple(lockTags, ShareLock, true); CommitTransactionCommand(); forboth(lc, indexIds, lc2, newIndexIds) @@ -3009,7 +3030,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) * for more details. */ - WaitForLockersMultiple(lockTags, ShareLock, false); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_2); + WaitForLockersMultiple(lockTags, ShareLock, true); CommitTransactionCommand(); foreach(lc, newIndexIds) @@ -3057,6 +3080,8 @@ ReindexRelationConcurrently(Oid relationOid, int options) * before the reference snap was taken, we have to wait out any * transactions that might have older snapshots. */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_3); WaitForOlderSnapshots(limitXmin, false); CommitTransactionCommand(); @@ -3128,7 +3153,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) * index_drop() for more details. */ - WaitForLockersMultiple(lockTags, AccessExclusiveLock, false); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_4); + WaitForLockersMultiple(lockTags, AccessExclusiveLock, true); foreach(lc, indexIds) { @@ -3150,7 +3177,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) * Drop the old indexes. */ - WaitForLockersMultiple(lockTags, AccessExclusiveLock, false); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_4); + WaitForLockersMultiple(lockTags, AccessExclusiveLock, true); PushActiveSnapshot(GetTransactionSnapshot()); @@ -3225,6 +3254,8 @@ ReindexRelationConcurrently(Oid relationOid, int options) MemoryContextDelete(private_context); + pgstat_progress_end_command(); + return true; } diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index f046fa13b1..37043e926d 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -59,6 +59,7 @@ /* Progress parameters for CREATE INDEX */ /* 3, 4 and 5 reserved for "waitfor" metrics */ +#define PROGRESS_CREATEIDX_INDEX_OID 6 #define PROGRESS_CREATEIDX_ACCESS_METHOD_OID 8 #define PROGRESS_CREATEIDX_PHASE 9 /* AM-agnostic phase # */ #define PROGRESS_CREATEIDX_SUBPHASE 10 /* phase # filled by AM */ @@ -76,6 +77,8 @@ #define PROGRESS_CREATEIDX_PHASE_VALIDATE_SORT 5 #define PROGRESS_CREATEIDX_PHASE_VALIDATE_TABLESCAN 6 #define PROGRESS_CREATEIDX_PHASE_WAIT_3 7 +#define PROGRESS_CREATEIDX_PHASE_WAIT_4 8 +#define PROGRESS_CREATEIDX_PHASE_WAIT_5 9 /* * Subphases of CREATE INDEX, for index_build. diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index bf7fca54ee..4638374a76 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1855,7 +1855,7 @@ pg_stat_progress_cluster| SELECT s.pid, WHEN 7 THEN 'performing final cleanup'::text ELSE NULL::text END AS phase, - s.param3 AS cluster_index_relid, + (s.param3)::oid AS cluster_index_relid, s.param4 AS heap_tuples_scanned, s.param5 AS heap_tuples_written, s.param6 AS heap_blks_total, @@ -1867,15 +1867,18 @@ pg_stat_progress_create_index| SELECT s.pid, s.datid, d.datname, s.relid, + (s.param7)::oid AS index_relid, CASE s.param10 WHEN 0 THEN 'initializing'::text - WHEN 1 THEN 'waiting for old snapshots'::text + WHEN 1 THEN 'waiting for writers before build'::text WHEN 2 THEN ('building index'::text || COALESCE((': '::text || pg_indexam_progress_phasename((s.param9)::oid, s.param11)), ''::text)) - WHEN 3 THEN 'waiting for writer snapshots'::text + WHEN 3 THEN 'waiting for writers before validation'::text WHEN 4 THEN 'index validation: scanning index'::text WHEN 5 THEN 'index validation: sorting tuples'::text WHEN 6 THEN 'index validation: scanning table'::text - WHEN 7 THEN 'waiting for reader snapshots'::text + WHEN 7 THEN 'waiting for old snapshots'::text + WHEN 8 THEN 'waiting for readers before marking dead'::text + WHEN 9 THEN 'waiting for readers before dropping'::text ELSE NULL::text END AS phase, s.param4 AS lockers_total, -- 2.21.0