From 2c455d62aad84267e987b07a1287fd979abc0995 Mon Sep 17 00:00:00 2001 From: Atsushi Torikoshi Date: Wed, 25 Sep 2024 21:28:15 +0900 Subject: [PATCH v7 1/3] Add log_verbosity = 'silent' support to COPY command. Previously, when the on_error option was set to ignore, the COPY command would always log NOTICE messages for input rows discarded due to data type incompatibility. Users had no way to suppress these messages. This commit introduces a new log_verbosity setting, 'silent', which prevents the COPY command from emitting NOTICE messages when on_error = 'ignore' is used, even if rows are discarded. This feature is particularly useful when processing malformed files frequently, where a flood of NOTICE messages can be undesirable. For example, when frequently loading malformed files via the COPY command or querying foreign tables using file_fdw (with an upcoming patch to add on_error support for file_fdw), users may prefer to suppress these messages to reduce log noise and improve clarity. Author: Atsushi Torikoshi Reviewed-by: Masahiko Sawada, Fujii Masao Discussion: https://postgr.es/m/ab59dad10490ea3734cf022b16c24cfd@oss.nttdata.com --- doc/src/sgml/ref/copy.sgml | 10 +++++++--- src/backend/commands/copy.c | 4 +++- src/backend/commands/copyfrom.c | 3 ++- src/bin/psql/tab-complete.c | 2 +- src/include/commands/copy.h | 4 +++- src/test/regress/expected/copy2.out | 4 +++- src/test/regress/sql/copy2.sql | 4 ++++ 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index fdbd20bc50..58a14bc427 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -407,6 +407,8 @@ COPY { table_name [ ( verbose, a NOTICE message containing the line of the input file and the column name whose input conversion has failed is emitted for each discarded row. + When it is set to silent, no message is emitted + regarding ignored rows. @@ -428,9 +430,11 @@ COPY { table_name [ ( Specify the amount of messages emitted by a COPY - command: default or verbose. If - verbose is specified, additional messages are emitted - during processing. + command: default, verbose, or + silent. + If verbose is specified, additional messages are + emitted during processing. + silent suppresses both verbose and default messages. This is currently used in COPY FROM command when diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3bb579a3a4..03eb7a4eba 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -427,9 +427,11 @@ defGetCopyLogVerbosityChoice(DefElem *def, ParseState *pstate) char *sval; /* - * Allow "default", or "verbose" values. + * Allow "silent", "default", or "verbose" values. */ sval = defGetString(def); + if (pg_strcasecmp(sval, "silent") == 0) + return COPY_LOG_VERBOSITY_SILENT; if (pg_strcasecmp(sval, "default") == 0) return COPY_LOG_VERBOSITY_DEFAULT; if (pg_strcasecmp(sval, "verbose") == 0) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 2d3462913e..47879994f7 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1320,7 +1320,8 @@ CopyFrom(CopyFromState cstate) error_context_stack = errcallback.previous; if (cstate->opts.on_error != COPY_ON_ERROR_STOP && - cstate->num_errors > 0) + cstate->num_errors > 0 && + cstate->opts.log_verbosity >= COPY_LOG_VERBOSITY_DEFAULT) ereport(NOTICE, errmsg_plural("%llu row was skipped due to data type incompatibility", "%llu rows were skipped due to data type incompatibility", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index a7ccde6d7d..6530b0f1ce 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2916,7 +2916,7 @@ psql_completion(const char *text, int start, int end) /* Complete COPY FROM filename WITH (LOG_VERBOSITY */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "LOG_VERBOSITY")) - COMPLETE_WITH("default", "verbose"); + COMPLETE_WITH("silent", "default", "verbose"); /* Complete COPY FROM WITH () */ else if (Matches("COPY|\\copy", MatchAny, "FROM", MatchAny, "WITH", MatchAny)) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 141fd48dc1..6f64d97fdd 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -45,7 +45,9 @@ typedef enum CopyOnErrorChoice */ typedef enum CopyLogVerbosityChoice { - COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages, default */ + COPY_LOG_VERBOSITY_SILENT = -1, /* logs none */ + COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages. As this is + * the default, assign 0 */ COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */ } CopyLogVerbosityChoice; diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 61a19cdc4c..4e752977b5 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -760,6 +760,7 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose); NOTICE: skipping row due to data type incompatibility at line 2 for column "l": null input CONTEXT: COPY check_ign_err2 NOTICE: 1 row was skipped due to data type incompatibility +COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent); -- reset context choice \set SHOW_CONTEXT errors SELECT * FROM check_ign_err; @@ -774,7 +775,8 @@ SELECT * FROM check_ign_err2; n | m | k | l ---+-----+---+------- 1 | {1} | 1 | 'foo' -(1 row) + 3 | {3} | 3 | 'bar' +(2 rows) -- test datatype error that can't be handled as soft: should fail CREATE TABLE hard_err(foo widget); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 8b14962194..fa6aa17344 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -533,6 +533,10 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose); 1 {1} 1 'foo' 2 {2} 2 \N \. +COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent); +3 {3} 3 'bar' +4 {4} 4 \N +\. -- reset context choice \set SHOW_CONTEXT errors -- 2.45.2