Merge branch 'main' into wrj/expr-bench

Cargo.toml

@@ -23,6 +23,8 @@ members = [
   "src/sqlparser",
   "src/sqlparser/test_runner",
   "src/storage",
+  "src/storage/backup",
+  "src/storage/backup/cmd",
   "src/storage/compactor",
   "src/storage/hummock_sdk",
   "src/storage/hummock_test",

Makefile.toml

@@ -11,6 +11,7 @@ extend = [
   { path = "src/risedevtool/risedev-components.toml" },
   { path = "src/frontend/planner_test/planner_test.toml" },
   { path = "src/tests/compaction_test/Makefile.toml" },
+  { path = "src/storage/backup/integration_tests/Makefile.toml" },
 ]
 
 env_files = ["./risedev-components.user.env"]

ci/scripts/build.sh

@@ -40,12 +40,14 @@ cargo build \
     -p risingwave_regress_test \
     -p risingwave_sqlsmith \
     -p risingwave_compaction_test \
+    -p risingwave_backup_cmd \
     --features "static-link static-log-level" --profile "$profile"
 
 echo "--- Compress debug info for artifacts"
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/sqlsmith
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/compaction-test
+objcopy --compress-debug-sections=zlib-gnu target/"$target"/backup-restore
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave_regress_test
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/risedev-dev
 
@@ -54,6 +56,7 @@ ldd target/"$target"/risingwave
 
 echo "--- Upload artifacts"
 cp target/"$target"/compaction-test ./compaction-test-"$profile"
+cp target/"$target"/backup-restore ./backup-restore-"$profile"
 cp target/"$target"/risingwave ./risingwave-"$profile"
 cp target/"$target"/risedev-dev ./risedev-dev-"$profile"
 cp target/"$target"/risingwave_regress_test ./risingwave_regress_test-"$profile"
@@ -63,3 +66,4 @@ buildkite-agent artifact upload risedev-dev-"$profile"
 buildkite-agent artifact upload risingwave_regress_test-"$profile"
 buildkite-agent artifact upload ./sqlsmith-"$profile"
 buildkite-agent artifact upload ./compaction-test-"$profile"
+buildkite-agent artifact upload ./backup-restore-"$profile"

ci/scripts/cron-e2e-test.sh

@@ -7,4 +7,5 @@ source ci/scripts/common.env.sh
 export RUN_SQLSMITH=1; # fuzz tests
 export SQLSMITH_COUNT=1000; # fuzz with 1000 queries
 export RUN_COMPACTION=1;
+export RUN_META_BACKUP=1;
 source ci/scripts/run-e2e-test.sh

ci/scripts/pr.env.sh

@@ -13,6 +13,8 @@ set -e
 
 # Don't run e2e compaction test in PR build
 export RUN_COMPACTION=0;
+# Don't run meta store backup/recovery test
+export RUN_META_BACKUP=0;
 
 if [[ -n "$CHANGED" ]]; then
     echo "Changes to Sqlsmith source files detected.";

ci/scripts/run-e2e-test.sh

@@ -70,6 +70,18 @@ sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/extended_query/**/*
 echo "--- Kill cluster"
 cargo make ci-kill
 
+if [[ "$RUN_META_BACKUP" -eq "1" ]]; then
+    echo "--- e2e, ci-meta-backup-test"
+    buildkite-agent artifact download backup-restore-"$profile" target/debug/
+    mv target/debug/backup-restore-"$profile" target/debug/backup-restore
+    chmod +x ./target/debug/backup-restore
+
+    test_root="src/storage/backup/integration_tests"
+    BACKUP_TEST_PREFIX_BIN="target/debug" BACKUP_TEST_PREFIX_DATA=".risingwave/data" bash "${test_root}/run_all.sh"
+    echo "--- Kill cluster"
+    cargo make kill
+fi
+
 if [[ "$RUN_COMPACTION" -eq "1" ]]; then
     echo "--- e2e, ci-compaction-test, nexmark_q7"
     cargo make clean-data