From c83f3ae09bacb961c511ebe9cb1f9f79bf1e3d29 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 13 Mar 2018 12:22:24 -0400 Subject: [PATCH 1/1] Exclude temporary relations from base backup. Exclude temporary relations during a base backup using the existing looks_like_temp_rel_name() function for identification. It shares code to identify database directories with [1], so for now that has been duplicated in this patch to make it independent. I'll rebase depending on what gets committed first. [1] https://www.postgresql.org/message-id/04791bab-cb04-ba43-e9c0-664a4c1ffb2c%40pgmasters.net --- doc/src/sgml/protocol.sgml | 2 +- src/backend/replication/basebackup.c | 41 +++++++++++++++++++++++ src/backend/storage/file/fd.c | 3 +- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 49 +++++++++++++++++++++++++++- src/include/storage/fd.h | 1 + 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 4fd61d7c2d..629a462574 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2565,7 +2565,7 @@ The commands accepted in replication mode are: Various temporary files and directories created during the operation of the PostgreSQL server, such as any file or directory beginning - with pgsql_tmp. + with pgsql_tmp and temporary relations. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 185f32a5f9..f0c3d13b2b 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -26,6 +26,7 @@ #include "nodes/pg_list.h" #include "pgtar.h" #include "pgstat.h" +#include "port.h" #include "postmaster/syslogger.h" #include "replication/basebackup.h" #include "replication/walsender.h" @@ -958,6 +959,36 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, char pathbuf[MAXPGPATH * 2]; struct stat statbuf; int64 size = 0; + const char *lastDir; /* Split last dir from parent path. */ + bool isDbDir = false; /* Does this directory contain relations? */ + + /* + * Determine if the current path is a database directory that can + * contain relations. + * + * Start by finding the location of the delimiter between the parent + * path and the current path. + */ + lastDir = last_dir_separator(path); + + /* Does this path look like a database path (i.e. all digits)? */ + if (lastDir != NULL && + strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1)) + { + /* Part of path that contains the parent directory. */ + int parentPathLen = lastDir - path; + + /* + * Mark path as a database directory if the parent path is either + * $PGDATA/base or a tablespace version path. + */ + if (strncmp(path, "./base", parentPathLen) == 0 || + (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) && + strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1), + TABLESPACE_VERSION_DIRECTORY, + sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0)) + isDbDir = true; + } dir = AllocateDir(path); while ((de = ReadDir(dir, path)) != NULL) @@ -1007,6 +1038,16 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, if (excludeFound) continue; + /* Exclude temporary relations */ + if (isDbDir && looks_like_temp_rel_name(de->d_name)) + { + elog(DEBUG2, + "temporary relation file \"%s\" excluded from backup", + de->d_name); + + continue; + } + snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name); /* Skip pg_control here to back up it last */ diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 2a18e94ff4..d30a725f90 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -325,7 +325,6 @@ static void RemovePgTempFilesInDir(const char *tmpdirname, bool missing_ok, bool unlink_all); static void RemovePgTempRelationFiles(const char *tsdirname); static void RemovePgTempRelationFilesInDbspace(const char *dbspacedirname); -static bool looks_like_temp_rel_name(const char *name); static void walkdir(const char *path, void (*action) (const char *fname, bool isdir, int elevel), @@ -3192,7 +3191,7 @@ RemovePgTempRelationFilesInDbspace(const char *dbspacedirname) } /* t_, or t__ */ -static bool +bool looks_like_temp_rel_name(const char *name) { int pos; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index cdf4f5be37..096472fd0d 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -2,9 +2,10 @@ use strict; use warnings; use Cwd; use Config; +use File::Basename qw(basename dirname); use PostgresNode; use TestLib; -use Test::More tests => 79; +use Test::More tests => 85; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -66,6 +67,18 @@ foreach my $filename ( # positive. $node->safe_psql('postgres', 'SELECT 1;'); +# Create files that look like temporary relations to ensure they are ignored. +my $postgresOid = $node->safe_psql('postgres', + q{select oid from pg_database where datname = 'postgres'}); + +my @tempRelationFiles = qw(t999_999 t9999_999.1 t999_9999_vm t99999_99999_vm.1); + +foreach my $filename (@tempRelationFiles) +{ + append_to_file("$pgdata/base/$postgresOid/$filename", 'TEMP_RELATION'); +} + +# Run base backup. $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup", '-X', 'none' ], 'pg_basebackup runs'); ok(-f "$tempdir/backup/PG_VERSION", 'backup was created'); @@ -96,6 +109,13 @@ foreach my $filename ( ok(!-f "$tempdir/backup/$filename", "$filename not copied"); } +# Temp relations should not be copied. +foreach my $filename (@tempRelationFiles) +{ + ok(!-f "$tempdir/backup/base/$postgresOid/$filename", + "base/$postgresOid/$filename not copied"); +} + # Make sure existing backup_label was ignored. isnt(slurp_file("$tempdir/backup/backup_label"), 'DONOTCOPY', 'existing backup_label not copied'); @@ -177,6 +197,19 @@ SKIP: my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar"; is(scalar(@tblspc_tars), 1, 'one tablespace tar was created'); + # Create files that look like temporary relations to ensure they are ignored + # in a tablespace. + my @tempRelationFiles = qw(t888_888 t888888_888888_vm.1); + my $tblSpc1Id = basename(dirname(dirname($node->safe_psql('postgres', + q{select pg_relation_filepath('test1')})))); + + foreach my $filename (@tempRelationFiles) + { + append_to_file( + "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename", + 'TEMP_RELATION'); + } + $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ], 'plain format with tablespaces fails without tablespace mapping'); @@ -195,6 +228,20 @@ SKIP: "tablespace symlink was updated"); closedir $dh; + # Temp relations should not be copied. + foreach my $filename (@tempRelationFiles) + { + ok(!-f "$tempdir/tbackup/tblspc1/$tblSpc1Id/$postgresOid/$filename", + "[tblspc1]/$postgresOid/$filename not copied"); + + # Also remove temp relation files or tablespace drop will fail. + my $filepath = + "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename"; + + unlink($filepath) + or BAIL_OUT("unable to unlink $filepath"); + } + ok( -d "$tempdir/backup1/pg_replslot", 'pg_replslot symlink copied as directory'); diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 4244e7b1fd..e49b42ce86 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -124,6 +124,7 @@ extern void AtEOXact_Files(void); extern void AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid, SubTransactionId parentSubid); extern void RemovePgTempFiles(void); +extern bool looks_like_temp_rel_name(const char *name); extern int pg_fsync(int fd); extern int pg_fsync_no_writethrough(int fd); -- 2.14.3 (Apple Git-98)