Meta upgrade (#5174)

* Meta upgrade remove all fulltext index when upgrade from V3 to V3_4 because of refacting of fulltext index * fix bug Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>
vesoft-inc · Jan 5, 2023 · 404880f · 404880f
1 parent cfa0475
commit 404880f
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 92 deletions.
diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h
@@ -774,7 +774,7 @@ class MetaClient : public BaseMetaClient {
   std::atomic<int64_t> metadLastUpdateTime_{0};
 
   int64_t metaServerVersion_{-1};
-  static constexpr int64_t EXPECT_META_VERSION = 3;
+  static constexpr int64_t EXPECT_META_VERSION = 4;
 
   // leadersLock_ is used to protect leadersInfo
   folly::SharedMutex leadersLock_;

diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp
@@ -147,16 +147,18 @@ std::unique_ptr<nebula::kvstore::KVStore> initKV(std::vector<nebula::HostAddr> p
     LOG(ERROR) << "Meta version is invalid";
     return nullptr;
   } else if (version == nebula::meta::MetaVersion::V1) {
-    LOG(ERROR) << "Can't upgrade meta from V1 to V3";
+    LOG(ERROR) << "Can't upgrade meta from V1 to V3_4";
     return nullptr;
   } else if (version == nebula::meta::MetaVersion::V2) {
-    auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine);
+    LOG(ERROR) << "Can't upgrade meta from V2 to V3_4";
+    return nullptr;
+  } else if (version == nebula::meta::MetaVersion::V3) {
+    auto ret = nebula::meta::MetaVersionMan::updateMetaV3ToV3_4(engine);
     if (!ret.ok()) {
-      LOG(ERROR) << "Update meta from V2 to V3 failed " << ret;
+      LOG(ERROR) << "Update meta from V3 to V3_4 failed " << ret;
       return nullptr;
     }
-
-    nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V3);
+    nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V3_4);
   }
 
   LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId;

diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp
@@ -43,10 +43,10 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) {
   }
   if (iter->valid()) {
     auto v1KeySize = hostPrefix.size() + sizeof(int64_t);
-    return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3;
+    return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3_4;
   }
   // No hosts exists, regard as version 3
-  return MetaVersion::V3;
+  return MetaVersion::V3_4;
 }
 
 // static
@@ -58,7 +58,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion v
   return code == nebula::cpp2::ErrorCode::SUCCEEDED;
 }
 
-Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
+Status MetaVersionMan::updateMetaV3ToV3_4(kvstore::KVEngine* engine) {
   CHECK_NOTNULL(engine);
   auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr());
 
@@ -75,7 +75,7 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
     return Status::Error("Create snapshot failed");
   }
 
-  auto status = doUpgradeV2ToV3(engine);
+  auto status = doUpgradeV3ToV3_4(engine);
   if (!status.ok()) {
     // rollback by snapshot
     return status;
@@ -89,87 +89,25 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
   return Status::OK();
 }
 
-Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVEngine* engine) {
-  MetaDataUpgrade upgrader(engine);
-  // Step 1: Upgrade HeartBeat into machine list
-  {
-    // collect all hosts association with zone
-    std::vector<HostAddr> zoneHosts;
-    const auto& zonePrefix = MetaKeyUtils::zonePrefix();
-    std::unique_ptr<kvstore::KVIterator> zoneIter;
-    auto code = engine->prefix(zonePrefix, &zoneIter);
-    if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
-      LOG(INFO) << "Get active hosts failed";
-      return Status::Error("Get hosts failed");
-    }
-
-    while (zoneIter->valid()) {
-      auto hosts = MetaKeyUtils::parseZoneHosts(zoneIter->val());
-      if (!hosts.empty()) {
-        zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end());
-      }
-      zoneIter->next();
-    }
-
-    const auto& prefix = MetaKeyUtils::hostPrefix();
-    std::unique_ptr<kvstore::KVIterator> iter;
-    code = engine->prefix(prefix, &iter);
-    if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
-      LOG(INFO) << "Get active hosts failed";
-      return Status::Error("Get hosts failed");
-    }
-
-    std::vector<kvstore::KV> data;
-    while (iter->valid()) {
-      auto info = HostInfo::decode(iter->val());
-
-      if (info.role_ == meta::cpp2::HostRole::STORAGE) {
-        // Save the machine information
-        auto host = MetaKeyUtils::parseHostKey(iter->key());
-        auto machineKey = MetaKeyUtils::machineKey(host.host, host.port);
-        data.emplace_back(std::move(machineKey), "");
-
-        auto hostIt = std::find(zoneHosts.begin(), zoneHosts.end(), host);
-        if (hostIt == zoneHosts.end()) {
-          // Save the zone information
-          auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port);
-          auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName));
-          auto zoneVal = MetaKeyUtils::zoneVal({host});
-          data.emplace_back(std::move(zoneKey), std::move(zoneVal));
-        }
-      }
-      iter->next();
-    }
-    auto status = upgrader.saveMachineAndZone(std::move(data));
-    if (!status.ok()) {
-      LOG(INFO) << status;
-      return status;
-    }
+Status MetaVersionMan::doUpgradeV3ToV3_4(kvstore::KVEngine* engine) {
+  std::unique_ptr<kvstore::KVIterator> fulltextIter;
+  auto code = engine->prefix(MetaKeyUtils::fulltextIndexPrefix(), &fulltextIter);
+  if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
+    LOG(ERROR) << "Upgrade meta failed";
+    return Status::Error("Update meta failed");
   }
-
-  // Step 2: Update Create space properties about Group
-  {
-    const auto& prefix = MetaKeyUtils::spacePrefix();
-    std::unique_ptr<kvstore::KVIterator> iter;
-    auto code = engine->prefix(prefix, &iter);
-    if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
-      LOG(INFO) << "Get spaces failed";
-      return Status::Error("Get spaces failed");
-    }
-
-    while (iter->valid()) {
-      if (FLAGS_print_info) {
-        upgrader.printSpacesV2(iter->val());
-      }
-      auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val());
-      if (!status.ok()) {
-        LOG(INFO) << status;
-        return status;
-      }
-      iter->next();
-    }
+  std::vector<std::string> fulltextList;
+  while (fulltextIter->valid()) {
+    fulltextList.push_back(fulltextIter->key().toString());
+    fulltextIter->next();
   }
-  if (!setMetaVersionToKV(engine, MetaVersion::V3)) {
+  code = engine->multiRemove(fulltextList);
+  if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
+    LOG(ERROR) << "Upgrade meta failed";
+    return Status::Error("Upgrade meta failed");
+  }
+
+  if (!setMetaVersionToKV(engine, MetaVersion::V3_4)) {
     return Status::Error("Persist meta version failed");
   } else {
     return Status::OK();

diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h
@@ -19,6 +19,7 @@ enum class MetaVersion {
   V1 = 1,
   V2 = 2,
   V3 = 3,
+  V3_4 = 4,
 };
 
 /**
@@ -38,12 +39,12 @@ class MetaVersionMan final {
 
   static bool setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version);
 
-  static Status updateMetaV2ToV3(kvstore::KVEngine* engine);
+  static Status updateMetaV3ToV3_4(kvstore::KVEngine* engine);
 
  private:
   static MetaVersion getVersionByHost(kvstore::KVStore* kv);
 
-  static Status doUpgradeV2ToV3(kvstore::KVEngine* engine);
+  static Status doUpgradeV3ToV3_4(kvstore::KVEngine* engine);
 };
 
 }  // namespace meta

diff --git a/src/tools/meta-dump/MetaDumpTool.cpp b/src/tools/meta-dump/MetaDumpTool.cpp
@@ -42,6 +42,7 @@ class MetaDumper {
         V1 = 1,
         V2 = 2,
         V3 = 3,
+        V3_4 = 4,
       };
 
       prefix = "__meta_version__";
@@ -60,7 +61,7 @@ class MetaDumper {
         while (iter->Valid() && iter->key().starts_with(prefix)) {
           found = true;
           auto v1KeySize = prefix.size() + sizeof(int64_t);
-          auto version = (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3;
+          auto version = (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3_4;
           LOG(INFO) << "Meta version=" << static_cast<int>(version);
           iter->Next();
           break;