From 146018b9b78de2b858a97748217c4554eee28b3c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Jan 2025 14:20:52 +0100 Subject: [PATCH v12 2/2] Logical replication vs. virtual generated columns Commits 745217a051a and 7054186c4eb (with subsequent amendments in 5b0c46ea093, 8fcd80258bc, 87ce27de696) added support for using (stored) generated columns in logical replication. But this doesn't work for virtual generated columns, because their value is not available in logical decoding. This patch re-enables the original prohibitions, but this time for virtual generated columns only. The publication option "publish_generated_columns" and some related internal variables are not renamed, but maybe they should be. Discussion: https://www.postgresql.org/message-id/flat/a368248e-69e4-40be-9c07-6c3b5880b0a6@eisentraut.org --- doc/src/sgml/ddl.sgml | 4 +++- doc/src/sgml/logical-replication.sgml | 2 +- doc/src/sgml/ref/create_publication.sgml | 6 +++--- src/backend/catalog/pg_publication.c | 21 +++++++++++++++++---- src/backend/replication/logical/proto.c | 5 +++++ src/backend/replication/logical/relation.c | 4 ++-- src/backend/replication/logical/tablesync.c | 4 ++-- src/backend/replication/pgoutput/pgoutput.c | 2 +- src/test/regress/expected/publication.out | 9 +++++++-- src/test/regress/sql/publication.sql | 8 ++++++-- src/test/subscription/t/031_column_list.pl | 14 ++++++++------ 11 files changed, 55 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 502dd154645..16dc0212669 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -539,11 +539,13 @@ Generated Columns - Generated columns are allowed to be replicated during logical replication + Stored generated columns are allowed to be replicated during logical replication according to the CREATE PUBLICATION parameter publish_generated_columns or by including them in the column list of the CREATE PUBLICATION command. + Virtual generated columns are skipped for logical replication and cannot be + specified in a CREATE PUBLICATION column list. diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 8290cd1a083..83678da40a8 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -1438,7 +1438,7 @@ Column Lists copied. However, if the subscriber is from a release prior to 15, then all the columns in the table are copied during initial data synchronization, ignoring any column lists. If the subscriber is from a release prior to 18, - then initial table synchronization won't copy generated columns even if they + then initial table synchronization won't copy stored generated columns even if they are defined in the publisher. diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 5e25536554a..f61cea59725 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -89,7 +89,7 @@ Parameters When a column list is specified, only the named columns are replicated. - The column list can contain generated columns as well. If no column list + The column list can contain stored generated columns as well. If no column list is specified, all table columns (except generated columns) are replicated through this publication, including any columns added later. It has no effect on TRUNCATE commands. See @@ -193,7 +193,7 @@ Parameters publish_generated_columns (boolean) - Specifies whether the generated columns present in the tables + Specifies whether the stored generated columns present in the tables associated with the publication should be replicated. The default is false. @@ -201,7 +201,7 @@ Parameters If the subscriber is from a release prior to 18, then initial table - synchronization won't copy generated columns even if parameter + synchronization won't copy stored generated columns even if parameter publish_generated_columns is true in the publisher. diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index b89098f5e99..c56937d7204 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -546,7 +546,8 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, * pub_collist_validate * Process and validate the 'columns' list and ensure the columns are all * valid to use for a publication. Checks for and raises an ERROR for - * any unknown columns, system columns, or duplicate columns. + * any unknown columns, system columns, duplicate columns, or virtual + * generated columns. * * Looks up each column's attnum and returns a 0-based Bitmapset of the * corresponding attnums. @@ -556,6 +557,7 @@ pub_collist_validate(Relation targetrel, List *columns) { Bitmapset *set = NULL; ListCell *lc; + TupleDesc tupdesc = RelationGetDescr(targetrel); foreach(lc, columns) { @@ -574,6 +576,12 @@ pub_collist_validate(Relation targetrel, List *columns) errmsg("cannot use system column \"%s\" in publication column list", colname)); + if (TupleDescAttr(tupdesc, attnum - 1)->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) + ereport(ERROR, + errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("cannot use virtual generated column \"%s\" in publication column list", + colname)); + if (bms_is_member(attnum, set)) ereport(ERROR, errcode(ERRCODE_DUPLICATE_OBJECT), @@ -622,7 +630,8 @@ pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, MemoryContext mcxt) /* * Returns a bitmap representing the columns of the specified table. * - * Generated columns are included if include_gencols is true. + * Stored generated columns are included if include_gencols is true. Virtual + * generated columns are never included. */ Bitmapset * pub_form_cols_map(Relation relation, bool include_gencols) @@ -634,7 +643,9 @@ pub_form_cols_map(Relation relation, bool include_gencols) { Form_pg_attribute att = TupleDescAttr(desc, i); - if (att->attisdropped || (att->attgenerated && !include_gencols)) + if (att->attisdropped || + att->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL || + (att->attgenerated && !include_gencols)) continue; result = bms_add_member(result, att->attnum); @@ -1276,7 +1287,9 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) { Form_pg_attribute att = TupleDescAttr(desc, i); - if (att->attisdropped || (att->attgenerated && !pub->pubgencols)) + if (att->attisdropped || + att->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL || + (att->attgenerated && !pub->pubgencols)) continue; attnums[nattnums++] = att->attnum; diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index bef350714db..6b412019731 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -1260,6 +1260,8 @@ logicalrep_message_type(LogicalRepMsgType action) * * Note that generated columns can be published only when present in a * publication column list, or when include_gencols is true. + * + * Virtual generated columns are never published. */ bool logicalrep_should_publish_column(Form_pg_attribute att, Bitmapset *columns, @@ -1268,6 +1270,9 @@ logicalrep_should_publish_column(Form_pg_attribute att, Bitmapset *columns, if (att->attisdropped) return false; + if (att->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) + return false; + /* If a column list is provided, publish only the cols in that list. */ if (columns) return bms_is_member(att->attnum, columns); diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 67aab02ff76..1fea66a2604 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -455,8 +455,8 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->attrmap->attnums[i] = attnum; if (attnum >= 0) { - /* Remember which subscriber columns are generated. */ - if (attr->attgenerated) + /* Remember which subscriber columns are stored generated. */ + if (attr->attgenerated == ATTRIBUTE_GENERATED_STORED) generatedattrs = bms_add_member(generatedattrs, attnum); missingatts = bms_del_member(missingatts, attnum); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 6af5c9fe16c..68ab03a8e6b 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -947,9 +947,9 @@ fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel, " a.atttypid," " a.attnum = ANY(i.indkey)"); - /* Generated columns can be replicated since version 18. */ + /* Stored generated columns can be replicated since version 18. */ if (server_version >= 180000) - appendStringInfo(&cmd, ", a.attgenerated != ''"); + appendStringInfo(&cmd, ", a.attgenerated = '%c'", ATTRIBUTE_GENERATED_STORED); appendStringInfo(&cmd, " FROM pg_catalog.pg_attribute a" diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 2ee48e1b4c0..4a9ce8243aa 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -1035,7 +1035,7 @@ check_and_init_gencol(PGOutputData *data, List *publications, { Form_pg_attribute att = TupleDescAttr(desc, i); - if (att->attgenerated) + if (att->attgenerated == ATTRIBUTE_GENERATED_STORED) { gencolpresent = true; break; diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 7cc405e002d..c11ffa5a889 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -727,7 +727,9 @@ CREATE PUBLICATION testpub_fortable FOR TABLE testpub_tbl1; CREATE PUBLICATION testpub_fortable_insert WITH (publish = 'insert'); RESET client_min_messages; CREATE TABLE testpub_tbl5 (a int PRIMARY KEY, b text, c text, - d int generated always as (a + length(b)) stored); + d int generated always as (a + length(b)) stored, + e int generated always as (a + length(b)) virtual +); -- error: column "x" does not exist ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, x); ERROR: column "x" of relation "testpub_tbl5" does not exist @@ -764,9 +766,12 @@ UPDATE testpub_tbl5 SET a = 1; ERROR: cannot update table "testpub_tbl5" DETAIL: Column list used by the publication does not cover the replica identity. ALTER PUBLICATION testpub_fortable DROP TABLE testpub_tbl5; --- ok: generated column "d" can be in the list too +-- ok: stored generated column "d" can be in the list too ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, d); ALTER PUBLICATION testpub_fortable DROP TABLE testpub_tbl5; +-- error: virtual generated column "e" can't be in list +ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, e); +ERROR: cannot use virtual generated column "e" in publication column list -- error: change the replica identity to "b", and column list to (a, c) -- then update fails, because (a, c) does not cover replica identity ALTER TABLE testpub_tbl5 REPLICA IDENTITY USING INDEX testpub_tbl5_b_key; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 3b03fb647f1..d05f21a8472 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -447,7 +447,9 @@ CREATE PUBLICATION testpub_fortable FOR TABLE testpub_tbl1; CREATE PUBLICATION testpub_fortable_insert WITH (publish = 'insert'); RESET client_min_messages; CREATE TABLE testpub_tbl5 (a int PRIMARY KEY, b text, c text, - d int generated always as (a + length(b)) stored); + d int generated always as (a + length(b)) stored, + e int generated always as (a + length(b)) virtual +); -- error: column "x" does not exist ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, x); -- error: replica identity "a" not included in the column list @@ -474,9 +476,11 @@ CREATE UNIQUE INDEX testpub_tbl5_b_key ON testpub_tbl5 (b, c); UPDATE testpub_tbl5 SET a = 1; ALTER PUBLICATION testpub_fortable DROP TABLE testpub_tbl5; --- ok: generated column "d" can be in the list too +-- ok: stored generated column "d" can be in the list too ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, d); ALTER PUBLICATION testpub_fortable DROP TABLE testpub_tbl5; +-- error: virtual generated column "e" can't be in list +ALTER PUBLICATION testpub_fortable ADD TABLE testpub_tbl5 (a, e); -- error: change the replica identity to "b", and column list to (a, c) -- then update fails, because (a, c) does not cover replica identity diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl index 7a535e76b08..b16dea9aa17 100644 --- a/src/test/subscription/t/031_column_list.pl +++ b/src/test/subscription/t/031_column_list.pl @@ -1202,14 +1202,16 @@ is( $result, qq(t t), 'check the number of columns in the old tuple'); -# TEST: Dropped columns are not considered for the column list, and generated -# columns are not replicated if they are not explicitly included in the column -# list. So, the publication having a column list except for those columns and a -# publication without any column list (aka all columns as part of the columns -# list) are considered to have the same column list. +# TEST: Dropped columns and virtual generated columns are not +# considered for the column list, and stored generated columns are not +# replicated if they are not explicitly included in the column +# list. So, the publication having a column list except for those +# columns and a publication without any column list (aka all columns +# as part of the columns list) are considered to have the same column +# list. $node_publisher->safe_psql( 'postgres', qq( - CREATE TABLE test_mix_4 (a int PRIMARY KEY, b int, c int, d int GENERATED ALWAYS AS (a + 1) STORED); + CREATE TABLE test_mix_4 (a int PRIMARY KEY, b int, c int, d int GENERATED ALWAYS AS (a + 1) STORED, e int GENERATED ALWAYS AS (a + 2) VIRTUAL); ALTER TABLE test_mix_4 DROP COLUMN c; CREATE PUBLICATION pub_mix_7 FOR TABLE test_mix_4 (a, b); -- 2.47.1