From 1fd29fa00c777b2c04683394f54261b446e46a61 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 5 Nov 2024 16:47:42 -0600 Subject: [PATCH v1 4/8] Add --no-sync-data-files flag to initdb. THIS IS A PROOF OF CONCEPT AND IS NOT READY FOR SERIOUS REVIEW. This new mode caused 'initdb --sync-only' to synchronize everything except for the database directories. It will be used in a follow-up commit that aims to reduce the duration of the data synchronization step in pg_upgrade's catalog-swap mode. --- src/bin/initdb/initdb.c | 9 ++++-- src/bin/pg_basebackup/pg_basebackup.c | 2 +- src/bin/pg_checksums/pg_checksums.c | 2 +- src/bin/pg_combinebackup/pg_combinebackup.c | 2 +- src/bin/pg_rewind/file_ops.c | 2 +- src/common/file_utils.c | 35 ++++++++++++++++----- src/include/common/file_utils.h | 3 +- 7 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 9a91830783..53c6e86a80 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -168,6 +168,7 @@ static bool data_checksums = true; static char *xlog_dir = NULL; static int wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024); static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC; +static bool sync_data_files = true; /* internal vars */ @@ -3183,6 +3184,7 @@ main(int argc, char *argv[]) {"icu-rules", required_argument, NULL, 18}, {"sync-method", required_argument, NULL, 19}, {"no-data-checksums", no_argument, NULL, 20}, + {"no-sync-data-files", no_argument, NULL, 21}, {NULL, 0, NULL, 0} }; @@ -3377,6 +3379,9 @@ main(int argc, char *argv[]) case 20: data_checksums = false; break; + case 21: + sync_data_files = false; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -3428,7 +3433,7 @@ main(int argc, char *argv[]) fputs(_("syncing data to disk ... "), stdout); fflush(stdout); - sync_pgdata(pg_data, PG_VERSION_NUM, sync_method); + sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files); check_ok(); return 0; } @@ -3491,7 +3496,7 @@ main(int argc, char *argv[]) { fputs(_("syncing data to disk ... "), stdout); fflush(stdout); - sync_pgdata(pg_data, PG_VERSION_NUM, sync_method); + sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files); check_ok(); } else diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index e41a6cfbda..43526e3246 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -2310,7 +2310,7 @@ BaseBackup(char *compression_algorithm, char *compression_detail, } else { - (void) sync_pgdata(basedir, serverVersion, sync_method); + (void) sync_pgdata(basedir, serverVersion, sync_method, true); } } diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index b86bc417c9..06ccaacfda 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -633,7 +633,7 @@ main(int argc, char *argv[]) if (do_sync) { pg_log_info("syncing data directory"); - sync_pgdata(DataDir, PG_VERSION_NUM, sync_method); + sync_pgdata(DataDir, PG_VERSION_NUM, sync_method, true); } pg_log_info("updating control file"); diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c index 5f1f62f1db..80a137be4e 100644 --- a/src/bin/pg_combinebackup/pg_combinebackup.c +++ b/src/bin/pg_combinebackup/pg_combinebackup.c @@ -420,7 +420,7 @@ main(int argc, char *argv[]) else { pg_log_debug("recursively fsyncing \"%s\"", opt.output); - sync_pgdata(opt.output, version * 10000, opt.sync_method); + sync_pgdata(opt.output, version * 10000, opt.sync_method, true); } } diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index 67a86bb4c5..ceb1c3ac6d 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -296,7 +296,7 @@ sync_target_dir(void) if (!do_sync || dry_run) return; - sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method); + sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method, true); } diff --git a/src/common/file_utils.c b/src/common/file_utils.c index dc90f35ae1..65cdf07ae7 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -94,7 +94,8 @@ do_syncfs(const char *path) void sync_pgdata(const char *pg_data, int serverVersion, - DataDirSyncMethod sync_method) + DataDirSyncMethod sync_method, + bool sync_data_files) { bool xlog_is_symlink; char pg_wal[MAXPGPATH]; @@ -184,10 +185,11 @@ sync_pgdata(const char *pg_data, * fsync the data directory and its contents. */ #ifdef PG_FLUSH_DATA_WORKS - walkdir(pg_data, pre_sync_fname, false, NULL); + walkdir(pg_data, pre_sync_fname, false, &sync_data_files); if (xlog_is_symlink) - walkdir(pg_wal, pre_sync_fname, false, NULL); - walkdir(pg_tblspc, pre_sync_fname, true, NULL); + walkdir(pg_wal, pre_sync_fname, false, &sync_data_files); + if (sync_data_files) + walkdir(pg_tblspc, pre_sync_fname, true, NULL); #endif /* @@ -200,10 +202,11 @@ sync_pgdata(const char *pg_data, * get fsync'd twice. That's not an expected case so we don't * worry about optimizing it. */ - walkdir(pg_data, fsync_fname, false, NULL); + walkdir(pg_data, fsync_fname, false, &sync_data_files); if (xlog_is_symlink) - walkdir(pg_wal, fsync_fname, false, NULL); - walkdir(pg_tblspc, fsync_fname, true, NULL); + walkdir(pg_wal, fsync_fname, false, &sync_data_files); + if (sync_data_files) + walkdir(pg_tblspc, fsync_fname, true, NULL); } break; } @@ -296,7 +299,23 @@ walkdir(const char *path, (*action) (subpath, false, arg); break; case PGFILETYPE_DIR: - walkdir(subpath, action, false, arg); + + /* + * XXX: Checking here for the "sync_data_files" case is quite + * hacky, but it's not clear how to do better. Another option + * would be to send "de" down to the function, but that would + * introduce a huge number of function pointer calls and + * directory reads that we are trying to avoid. + */ +#ifdef PG_FLUSH_DATA_WORKS + if ((action != pre_sync_fname && action != fsync_fname) || +#else + if (action != fsync_fname || +#endif + !arg || *((bool *) arg) || + strcmp(de->d_name, "base") != 0) + walkdir(subpath, action, false, arg); + break; default: diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index c328f56a85..3743caa63e 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -35,7 +35,8 @@ struct iovec; /* avoid including port/pg_iovec.h here */ #ifdef FRONTEND extern int fsync_fname(const char *fname, bool isdir, void *arg); extern void sync_pgdata(const char *pg_data, int serverVersion, - DataDirSyncMethod sync_method); + DataDirSyncMethod sync_method, + bool sync_data_files); extern void sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method); extern int durable_rename(const char *oldfile, const char *newfile); extern int fsync_parent_path(const char *fname); -- 2.39.5 (Apple Git-154)