From 9202557280a47cabb971a50ef87d8c12eb0be987 Mon Sep 17 00:00:00 2001 From: Haruka Takatsuka Date: Wed, 18 Dec 2024 14:57:02 +0900 Subject: [PATCH] Add --exclude-tmpdir option, which makes exclude pgsql_tmp and snapshots dirs. If there are many large file size temporary files or logical replication snapshot files in the database cluster directory, it will take longer to get a full backup. These files are not likely to be used after the restore. Therefore, we have added an option to exclude these files when obtaining full backups. --- dir.c | 2 ++ docs/index-ja.html | 16 +++++++++++++++- docs/index.html | 16 +++++++++++++++- expected/backup.out | 4 ++++ expected/option.out | 1 + pg_rman.c | 10 ++++++++++ sql/backup.sh | 18 +++++++++++++++--- 7 files changed, 62 insertions(+), 5 deletions(-) diff --git a/dir.c b/dir.c index 7d9ceba4..fc118847 100644 --- a/dir.c +++ b/dir.c @@ -27,6 +27,8 @@ const char *pgdata_exclude[] = NULL, /* arclog_path will be set later */ NULL, /* srvlog_path will be set later */ NULL, /* 'pg_tblspc' will be set later */ + NULL, /* 'pg_logical' will be set later */ + NULL, /* 'pgsql_tmp' will be set later */ NULL, /* sentinel */ }; diff --git a/docs/index-ja.html b/docs/index-ja.html index 748d49be..3e1b49ad 100755 --- a/docs/index-ja.html +++ b/docs/index-ja.html @@ -660,6 +660,12 @@

バックアップ・オプション

  • バックアップ済みのサーバログの保持期限を指定します。--keep-srvlog-files では保存ファイル数を、--keep-srvlog-days では保存日数を指定します。これらのオプションを明示的に指定した場合、閾値を超えたファイルがサーバログが保存されるディレクトリ (log_directory) から削除されます。 本オプションは、--with-serverlog、--srvlog-path を指定してバックアップを取得した場合に限ります。 これらのオプションを同時に指定した場合は両方の閾値を超えたファイルが削除されます。
    • +
  • --exclude-tmpdir + + +
    • @@ -1071,6 +1077,14 @@

    パラメータの指定方法

    アーカイブWALのリストア方法 環境変数、設定ファイルにはboolean型で指定 + + +–exclude-tmpdir +EXCLUDE_TMPDIR +指定可 +一時ファイルとロジカルスナップショットをバックアップから除外 +環境変数、設定ファイルにはboolean型で指定 + @@ -1472,4 +1486,4 @@

    関連項目

    Copyright (c) 2009-2023, NIPPON TELEGRAPH AND TELEPHONE CORPORATION

    - \ No newline at end of file + diff --git a/docs/index.html b/docs/index.html index d9719950..11c272fe 100755 --- a/docs/index.html +++ b/docs/index.html @@ -651,6 +651,12 @@

    Backup options

  • Specify how long backuped serverlog files will be kept. --keep-srvlog-files means number of files. --keep-srvlog-days means days to be kept. When you do backup, only files exceeded specified condition are deleted from server log directory (log_directory). This option works when you specify --with-serverlog and --srvlog-path options in backup command. If these two options are given toghether, pg_rman deletes files which are old enough against both conditions.
    • +
  • --exclude-tmpdir + + +
    • @@ -1062,6 +1068,14 @@

    Way to pass options

    how to restore archive WAL specify boolean type in environmental variable or configuration file + + +–exclude-tmpdir +EXCLUDE_TMPDIR +Yes +Exclude temporary files and logical snapshots from backup +specify boolean type in environmental variable or configuration file + @@ -1462,4 +1476,4 @@

    See Also

    Copyright (c) 2009-2023, NIPPON TELEGRAPH AND TELEPHONE CORPORATION

    - \ No newline at end of file + diff --git a/expected/backup.out b/expected/backup.out index 67c9692d..4bf8b766 100644 --- a/expected/backup.out +++ b/expected/backup.out @@ -74,3 +74,7 @@ INFO: Please execute 'pg_rman validate' to verify the files are correctly copied ERROR: could not start backup DETAIL: system identifier of target database is different from the one of initially configured database 10 +###### BACKUP COMMAND TEST-0011 ###### +###### full backup with --exclude-tmpdir option ###### +0 +3 diff --git a/expected/option.out b/expected/option.out index dd787476..c390d7c5 100644 --- a/expected/option.out +++ b/expected/option.out @@ -40,6 +40,7 @@ Backup options: --keep-srvlog-days=DAY keep serverlog modified in DAY days --standby-host=HOSTNAME standby host when taking backup from standby --standby-port=PORT standby port when taking backup from standby + --exclude-tmpdir exclude pgsql_tmp and pg_logical/snapshots Restore options: --recovery-target-time time stamp up to which recovery will proceed diff --git a/pg_rman.c b/pg_rman.c index db263aee..145015c5 100644 --- a/pg_rman.c +++ b/pg_rman.c @@ -17,6 +17,8 @@ const char *PROGRAM_VERSION = "1.3.16"; const char *PROGRAM_URL = "http://github.com/ossc-db/pg_rman"; const char *PROGRAM_ISSUES = "http://github.com/ossc-db/pg_rman/issues"; +const char *SNAPSHOTS_DIR = "snapshots"; +const char *PGSQL_TMP_DIR = "pgsql_tmp"; /* path configuration */ char *backup_path; @@ -32,6 +34,7 @@ bool check = false; /* directory configuration */ pgBackup current; +static bool exclude_tmpdir = false; /* backup configuration */ static bool smooth_checkpoint; @@ -97,6 +100,7 @@ static pgut_option options[] = { 's', 10, "recovery-target-timeline" , &target_tli_string, SOURCE_ENV }, { 's', 11, "recovery-target-action" , &target_action , SOURCE_ENV }, { 'b', 12, "hard-copy" , &is_hard_copy , SOURCE_ENV }, + { 'b', 15, "exclude-tmpdir", &exclude_tmpdir, SOURCE_ENV }, /* catalog options */ { 'b', 'a', "show-all" , &show_all }, { 0 } @@ -209,6 +213,11 @@ main(int argc, char *argv[]) pgdata_exclude[i++] = arclog_path; if (srvlog_path) pgdata_exclude[i++] = srvlog_path; + if (exclude_tmpdir) + { + pgdata_exclude[i++] = SNAPSHOTS_DIR; + pgdata_exclude[i++] = PGSQL_TMP_DIR; + } /* do actual operation */ if (pg_strcasecmp(cmd, "init") == 0) @@ -289,6 +298,7 @@ pgut_help(bool details) printf(_(" --keep-srvlog-days=DAY keep serverlog modified in DAY days\n")); printf(_(" --standby-host=HOSTNAME standby host when taking backup from standby\n")); printf(_(" --standby-port=PORT standby port when taking backup from standby\n")); + printf(_(" --exclude-tmpdir exclude pgsql_tmp and pg_logical/snapshots\n")); printf(_("\nRestore options:\n")); printf(_(" --recovery-target-time time stamp up to which recovery will proceed\n")); printf(_(" --recovery-target-xid transaction ID up to which recovery will proceed\n")); diff --git a/sql/backup.sh b/sql/backup.sh index aabca919..94d5d8b4 100644 --- a/sql/backup.sh +++ b/sql/backup.sh @@ -195,9 +195,9 @@ sleep 1 echo 'archive backup in the same situation but with --full-backup-on-error option' pg_rman backup -B ${BACKUP_PATH} -b archive -F -s -Z -p ${TEST_PGPORT} -d postgres;echo $? pg_rman validate -B ${BACKUP_PATH} --quiet -pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0011.log 2>&1 -grep OK ${TEST_BASE}/TEST-0011.log | grep FULL | wc -l -grep ERROR ${TEST_BASE}/TEST-0011.log | grep ARCH | wc -l +pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0009.log 2>&1 +grep OK ${TEST_BASE}/TEST-0009.log | grep FULL | wc -l +grep ERROR ${TEST_BASE}/TEST-0009.log | grep ARCH | wc -l echo '###### BACKUP COMMAND TEST-0010 ######' echo '###### failure in backup with different system identifier database ######' @@ -206,6 +206,18 @@ pg_ctl stop -m immediate > /dev/null 2>&1 init_database pg_rman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --quiet;echo $? +echo '###### BACKUP COMMAND TEST-0011 ######' +echo '###### full backup with --exclude-tmpdir option ######' +init_catalog +mkdir -p $PGDATA_PATH/base/pgsql_tmp +touch $PGDATA_PATH/base/pgsql_tmp/012345 +touch $PGDATA_PATH/pg_logical/snapshots/012345 +pg_rman backup -B ${BACKUP_PATH} -b full -s --exclude-tmpdir -p ${TEST_PGPORT} -d postgres --quiet;echo $? +pg_rman validate -B ${BACKUP_PATH} --quiet +pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0011.log 2>&1 +test -f ${BACKUP_PATH}/????????/??????/database/base/pgsql_tmp/012345 || echo OK >> ${TEST_BASE}/TEST-0011.log +test -f ${BACKUP_PATH}/????????/??????/database/pg_logical/snapshots/012345 || echo OK >> ${TEST_BASE}/TEST-0011.log +grep -c OK ${TEST_BASE}/TEST-0011.log # cleanup ## clean up the temporal test data