[yugabyte#9797] DocDB: Avoid catalog version going backward during online upgrade

Summary:
Before and during a major YSQL version upgrade, through the heartbeat mechanism, the master provides current catalog versions to tservers from the PG11 catalog versions table. Note that DDLs are not allowed during a YSQL major version upgrade, and the PG15 catalogs are being updated by being set to semantic equivalents of PG11. To the PG11-only clients (tservers), there is only one valid catalog version state and that is the state when the upgrade began.

While going through the upgrade flow, the C++ upgrade tests keep the cluster running while the YB master is switched out of upgrade mode. When the master is switched out of upgrade mode in this way, it switches to providing the current catalog versions from the PG15 catalog versions table, which is fresh and sets everything as version 1. Thus tservers see the version number going down and probabilistically crash themselves with the following error message:

  Ignoring ysql db 13245 catalog version update: new version too old. New: 1, Old: 2, ignored count: 31

Fix this incorrect state by copying the contents of the PG11 catalog versions table to PG15, right after the PG15 catalog is established, and before returning success of the catalog upgrade.

Test Plan:
Jenkins

On MacOS arm64:
  ./yb_build.sh release --cxx-test pg15_upgrade-test --gtest_filter=Pg15UpgradeTest.Schemas

Reviewers: hsunder

Subscribers: ybase, yql

Differential Revision: https://phorge.dev.yugabyte.com/D38961

Author: foucher

pg15_tests/common_upgrade.sh

@@ -13,24 +13,26 @@ common_pg15_flags="TEST_always_return_consensus_info_for_succeeded_rpc=false,pg_
 # implemented.
 common_tserver_flags='"ysql_pg_conf_csv=yb_enable_expression_pushdown=false"'
 
+pg11_enable_db_catalog_flag="allowed_preview_flags_csv=ysql_enable_db_catalog_version_mode,ysql_enable_db_catalog_version_mode=true"
+
 # Downloads, runs, and pushds the directory for pg11.
 # Sets $pg11path to the pg11 directory.
 run_and_pushd_pg11() {
   prefix="/tmp"
-  ybversion_pg11="2024.2.0.0"
-  ybbuild="957e0756c1234079d7bda1cee74d468e7157f11e"
+  ybversion_pg11="2.20.2.2"
+  ybbuild="b1"
   if [[ $OSTYPE = linux* ]]; then
-    arch="release-clang17-centos-x86_64"
+    arch="linux-x86_64"
     tarbin="tar"
   fi
   if [[ $OSTYPE = darwin* ]]; then
-    arch="release-clang-darwin-arm64"
+    arch="darwin-x86_64"
     tarbin="gtar"
   fi
   ybfilename_pg11="yugabyte-$ybversion_pg11-$ybbuild-$arch.tar.gz"
 
   if [ ! -f "$prefix"/"$ybfilename_pg11" ]; then
-    curl "https://s3.us-west-2.amazonaws.com/uploads.dev.yugabyte.com/local-provider-test/$ybversion_pg11/$ybfilename_pg11" \
+    curl "https://downloads.yugabyte.com/releases/$ybversion_pg11/$ybfilename_pg11" \
       -o "$prefix"/"$ybfilename_pg11"
   fi
 
@@ -42,7 +44,15 @@ run_and_pushd_pg11() {
 
   pg11path="$prefix/yugabyte-$ybversion_pg11"
   pushd "$pg11path"
-  yb_ctl_destroy_create --rf=3 --tserver_flags="$common_tserver_flags"
+  yb_ctl_destroy_create --rf=3
+  ysqlsh <<EOT
+  SET yb_non_ddl_txn_for_sys_tables_allowed=true;
+  SELECT yb_fix_catalog_version_table(true);
+  SET yb_non_ddl_txn_for_sys_tables_allowed = false;
+EOT
+  yb_ctl restart \
+  --tserver_flags="$common_tserver_flags,$pg11_enable_db_catalog_flag" \
+  --master_flags="$pg11_enable_db_catalog_flag"
 }
 
 upgrade_masters() {

pg15_tests/test_upgrade_rollback.sh

@@ -25,7 +25,15 @@ build/latest/bin/yb-admin --init_master_addrs=127.0.0.200:7100 list_tables inclu
 # Roll back
 # Restart node 2 tserver as PG11
 pushd $pg11path
-yb_ctl restart_node 2 --tserver_flags="$common_tserver_flags"
+# YB_TODO: Since D31087 isn't in the PG11 "from" build, we need to undo PG15's pg_data symlink
+# and restore the pg_data_11 directory to pg_data.
+# When D31087 is in the PG11 build, replace the lines from stop_node to start_node with:
+#   yb_ctl restart_node 2
+# D31087 is present in 2.21 onwards, 2024.1 onwards.
+yb_ctl stop_node 2
+rm "$data_dir/node-2/disk-1/pg_data"
+mv "$data_dir/node-2/disk-1/pg_data_11" "$data_dir/node-2/disk-1/pg_data"
+yb_ctl start_node 2 --tserver_flags="$pg11_enable_db_catalog_flag" --master_flags="$pg11_enable_db_catalog_flag"
 popd
 # Issue the rollback RPC
 echo rollback starting at $(date +"%r")

src/yb/integration-tests/upgrade-tests/pg15_upgrade_test_base.h

@@ -19,7 +19,7 @@ namespace yb {
 
 class Pg15UpgradeTestBase : public UpgradeTestBase {
  public:
-  Pg15UpgradeTestBase() : UpgradeTestBase(kBuild_2024_2_0_0) {}
+  Pg15UpgradeTestBase() : UpgradeTestBase(kBuild_2_20_2_4) {}
   virtual ~Pg15UpgradeTestBase() override = default;
 
   void SetUp() override;

src/yb/integration-tests/upgrade-tests/upgrade_test_base.cc

@@ -201,6 +201,10 @@ void AddUnDefOkAndSetFlag(
   flag_list.emplace_back(Format("--$0=$1", flag_name, flag_value));
 }
 
+void AddAllowedPreviewFlag(std::vector<std::string>& flag_list, const std::string& flag_name) {
+  AddFlagToCsvFlag(flag_list, "allowed_preview_flags_csv", flag_name);
+}
+
 void WaitForAutoFlagApply() { SleepFor(FLAGS_auto_flags_apply_delay_ms * 1ms + 3s); }
 
 Status SetYsqlMajorUpgradeFlagOnMasters(ExternalMiniCluster& cluster, bool enable) {
@@ -220,9 +224,9 @@ UpgradeTestBase::UpgradeTestBase(const std::string& from_version)
 }
 
 void UpgradeTestBase::SetUp() {
-  if (old_version_info_.version != kBuild_2024_2_0_0) {
+  if (old_version_info_.version != kBuild_2_20_2_4) {
     test_skipped_ = true;
-    GTEST_SKIP() << "PG15 upgrade is only supported from version " << kBuild_2024_2_0_0;
+    GTEST_SKIP() << "PG15 upgrade is only supported from version " << kBuild_2_20_2_4;
   }
 
   if (IsSanitizer()) {
@@ -268,6 +272,15 @@ Status UpgradeTestBase::StartClusterInOldVersion(const ExternalMiniClusterOption
   AddUnDefOkAndSetFlag(
       opts.extra_tserver_flags, "TEST_always_return_consensus_info_for_succeeded_rpc", "false");
 
+  // YB_TODO: Enable ysql_enable_db_catalog_version_mode since it is not major version upgrade
+  // safe.
+  if (old_version_info_.version == kBuild_2_20_2_4) {
+    AddAllowedPreviewFlag(opts.extra_master_flags, "ysql_enable_db_catalog_version_mode");
+    AddUnDefOkAndSetFlag(opts.extra_master_flags, "ysql_enable_db_catalog_version_mode", "true");
+    AddAllowedPreviewFlag(opts.extra_tserver_flags, "ysql_enable_db_catalog_version_mode");
+    AddUnDefOkAndSetFlag(opts.extra_tserver_flags, "ysql_enable_db_catalog_version_mode", "true");
+  }
+
   LOG(INFO) << "Starting cluster in version: " << old_version_info_.version;
 
   RETURN_NOT_OK(ExternalMiniClusterITestBase::StartCluster(opts));
@@ -296,6 +309,14 @@ Status UpgradeTestBase::StartClusterInOldVersion(const ExternalMiniClusterOption
     RETURN_NOT_OK(cluster_->AddAndSetExtraFlag("ysql_yb_enable_expression_pushdown", "false"));
   }
 
+  if (old_version_info_.version == kBuild_2_20_2_4) {
+    // YB_TODO: Remove when the min upgrade-from version is 2024.1+.
+    auto conn = VERIFY_RESULT(cluster_->ConnectToDB());
+    RETURN_NOT_OK(conn.Execute("SET yb_non_ddl_txn_for_sys_tables_allowed=true"));
+    RETURN_NOT_OK(conn.Fetch("SELECT yb_fix_catalog_version_table(true)"));
+    RETURN_NOT_OK(conn.Execute("SET yb_non_ddl_txn_for_sys_tables_allowed=false"));
+  }
+
   return Status::OK();
 }
 
@@ -616,6 +637,16 @@ Status UpgradeTestBase::RestartTServerInOldVersion(
     ExternalTabletServer& ts, bool wait_for_cluster_to_stabilize) {
   LOG(INFO) << "Restarting yb-tserver " << ts.id() << " in old version";
 
+  if (is_ysql_major_version_upgrade_ && old_version_info_.version == kBuild_2_20_2_4) {
+    // YB_TODO: Remove this once we switch to a newer version.
+    // 2.20 does not have the pg_data symlink changes so we need to manually delete the pg_data
+    // folder and use the pg_data_11.
+    auto env = Env::Default();
+    const auto pg_data_dir = JoinPathSegments(ts.GetRootDir(), "pg_data");
+    RETURN_NOT_OK(env->DeleteRecursively(pg_data_dir));
+    RETURN_NOT_OK(env->RenameFile(JoinPathSegments(ts.GetRootDir(), "pg_data_11"), pg_data_dir));
+  }
+
   RETURN_NOT_OK(RestartDaemonInVersion(ts, old_version_tserver_bin_path_));
 
   if (wait_for_cluster_to_stabilize) {