Support Cluster Management

.linters/cpp/checkKeyword.py

@@ -111,6 +111,10 @@
     'KW_EXPLAIN',
     'KW_UNWIND',
     'KW_CASE',
+    'KW_HOSTS',
+    'KW_ZONE',
+    'KW_ZONES',
+    'KW_RENAME',
 ]
 
 

src/clients/meta/MetaClient.cpp

@@ -780,8 +780,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("Part not existed!");
     case nebula::cpp2::ErrorCode::E_USER_NOT_FOUND:
       return Status::Error("User not existed!");
-    case nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND:
-      return Status::Error("Group not existed!");
+    case nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND:
+      return Status::Error("Machine not existed!");
     case nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND:
       return Status::Error("Zone not existed!");
     case nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND:
@@ -3036,17 +3036,65 @@ void MetaClient::loadLeader(const std::vector<cpp2::HostItem>& hostItems,
   }
 }
 
-folly::Future<StatusOr<bool>> MetaClient::addZone(std::string zoneName,
-                                                  std::vector<HostAddr> nodes) {
-  cpp2::AddZoneReq req;
-  req.set_zone_name(std::move(zoneName));
-  req.set_nodes(std::move(nodes));
+folly::Future<StatusOr<bool>> MetaClient::addHosts(std::vector<HostAddr> hosts) {
+  cpp2::AddHostsReq req;
+  req.set_hosts(std::move(hosts));
+
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_addHosts(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
+
+folly::Future<StatusOr<bool>> MetaClient::dropHosts(std::vector<HostAddr> hosts) {
+  cpp2::DropHostsReq req;
+  req.set_hosts(std::move(hosts));
+
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_dropHosts(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
+
+folly::Future<StatusOr<bool>> MetaClient::mergeZone(std::vector<std::string> zones,
+                                                    std::string zoneName) {
+  cpp2::MergeZoneReq req;
+  req.set_zone_name(zoneName);
+  req.set_zones(zones);
+  folly::Promise<StatusOr<bool>> promise;
+  auto future = promise.getFuture();
+  getResponse(
+      std::move(req),
+      [](auto client, auto request) { return client->future_mergeZone(request); },
+      [](cpp2::ExecResp&& resp) -> bool {
+        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
+      },
+      std::move(promise));
+  return future;
+}
 
+folly::Future<StatusOr<bool>> MetaClient::renameZone(std::string originalZoneName,
+                                                     std::string zoneName) {
+  cpp2::RenameZoneReq req;
+  req.set_original_zone_name(originalZoneName);
+  req.set_zone_name(zoneName);
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_addZone(request); },
+      [](auto client, auto request) { return client->future_renameZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
@@ -3070,33 +3118,35 @@ folly::Future<StatusOr<bool>> MetaClient::dropZone(std::string zoneName) {
   return future;
 }
 
-folly::Future<StatusOr<bool>> MetaClient::addHostIntoZone(HostAddr node, std::string zoneName) {
-  cpp2::AddHostIntoZoneReq req;
-  req.set_node(node);
+folly::Future<StatusOr<bool>> MetaClient::splitZone(
+    std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>>) {
+  cpp2::SplitZoneReq req;
   req.set_zone_name(zoneName);
-
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_addHostIntoZone(request); },
+      [](auto client, auto request) { return client->future_splitZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },
       std::move(promise));
   return future;
 }
 
-folly::Future<StatusOr<bool>> MetaClient::dropHostFromZone(HostAddr node, std::string zoneName) {
-  cpp2::DropHostFromZoneReq req;
-  req.set_node(node);
+folly::Future<StatusOr<bool>> MetaClient::addHostsIntoZone(std::vector<HostAddr> hosts,
+                                                           std::string zoneName,
+                                                           bool isNew) {
+  cpp2::AddHostsIntoZoneReq req;
+  req.set_hosts(hosts);
   req.set_zone_name(zoneName);
+  req.set_is_new(isNew);
 
   folly::Promise<StatusOr<bool>> promise;
   auto future = promise.getFuture();
   getResponse(
       std::move(req),
-      [](auto client, auto request) { return client->future_dropHostFromZone(request); },
+      [](auto client, auto request) { return client->future_addHostsIntoZone(request); },
       [](cpp2::ExecResp&& resp) -> bool {
         return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
       },

src/clients/meta/MetaClient.h

@@ -597,13 +597,22 @@ class MetaClient {
 
   StatusOr<LeaderInfo> getLeaderInfo();
 
-  folly::Future<StatusOr<bool>> addZone(std::string zoneName, std::vector<HostAddr> nodes);
+  folly::Future<StatusOr<bool>> addHosts(std::vector<HostAddr> hosts);
+
+  folly::Future<StatusOr<bool>> dropHosts(std::vector<HostAddr> hosts);
+
+  folly::Future<StatusOr<bool>> mergeZone(std::vector<std::string> zones, std::string zoneName);
+
+  folly::Future<StatusOr<bool>> renameZone(std::string originalZoneName, std::string zoneName);
 
   folly::Future<StatusOr<bool>> dropZone(std::string zoneName);
 
-  folly::Future<StatusOr<bool>> addHostIntoZone(HostAddr node, std::string zoneName);
+  folly::Future<StatusOr<bool>> splitZone(
+      std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>> zones);
 
-  folly::Future<StatusOr<bool>> dropHostFromZone(HostAddr node, std::string zoneName);
+  folly::Future<StatusOr<bool>> addHostsIntoZone(std::vector<HostAddr> hosts,
+                                                 std::string zoneName,
+                                                 bool isNew);
 
   folly::Future<StatusOr<std::vector<HostAddr>>> getZone(std::string zoneName);
 

src/common/graph/Response.h

@@ -35,7 +35,7 @@
   X(E_TAG_PROP_NOT_FOUND, -10)                                                \
   X(E_ROLE_NOT_FOUND, -11)                                                    \
   X(E_CONFIG_NOT_FOUND, -12)                                                  \
-  X(E_GROUP_NOT_FOUND, -13)                                                   \
+  X(E_MACHINE_NOT_FOUND, -13)                                                 \
   X(E_ZONE_NOT_FOUND, -14)                                                    \
   X(E_LISTENER_NOT_FOUND, -15)                                                \
   X(E_PART_NOT_FOUND, -16)                                                    \

src/common/utils/MetaKeyUtils.cpp

@@ -21,6 +21,7 @@ static const std::unordered_map<std::string, std::pair<std::string, bool>> syste
     {"users", {"__users__", true}},
     {"hosts", {"__hosts__", false}},
     {"versions", {"__versions__", false}},
+    {"machines", {"__machines__", false}},
     {"snapshots", {"__snapshots__", false}},
     {"configs", {"__configs__", true}},
     {"groups", {"__groups__", true}},
@@ -58,8 +59,9 @@ static const std::unordered_map<
 // clang-format off
 static const std::string kSpacesTable         = tableMaps.at("spaces").first;         // NOLINT
 static const std::string kPartsTable          = tableMaps.at("parts").first;          // NOLINT
-static const std::string kHostsTable          = systemTableMaps.at("hosts").first;          // NOLINT
 static const std::string kVersionsTable       = systemTableMaps.at("versions").first; // NOLINT
+static const std::string kHostsTable          = systemTableMaps.at("hosts").first;    // NOLINT
+static const std::string kMachinesTable       = systemTableMaps.at("machines").first; // NOLINT
 static const std::string kTagsTable           = tableMaps.at("tags").first;           // NOLINT
 static const std::string kEdgesTable          = tableMaps.at("edges").first;          // NOLINT
 static const std::string kIndexesTable        = tableMaps.at("indexes").first;        // NOLINT
@@ -239,6 +241,21 @@ std::vector<HostAddr> MetaKeyUtils::parsePartValV2(folly::StringPiece val) {
   return ret;
 }
 
+std::string MetaKeyUtils::machineKey(std::string addr, Port port) {
+  std::string key;
+  HostAddr h(addr, port);
+  key.append(kMachinesTable.data(), kMachinesTable.size())
+      .append(MetaKeyUtils::serializeHostAddr(h));
+  return key;
+}
+
+const std::string& MetaKeyUtils::machinePrefix() { return kMachinesTable; }
+
+HostAddr MetaKeyUtils::parseMachineKey(folly::StringPiece key) {
+  key.advance(kMachinesTable.size());
+  return MetaKeyUtils::deserializeHostAddr(key);
+}
+
 std::string MetaKeyUtils::hostKey(std::string addr, Port port) { return hostKeyV2(addr, port); }
 
 std::string MetaKeyUtils::hostKeyV2(std::string addr, Port port) {
@@ -961,29 +978,6 @@ MetaKeyUtils::parseBalanceTaskVal(const folly::StringPiece& rawVal) {
   return std::make_tuple(status, ret, start, end);
 }
 
-std::string MetaKeyUtils::groupKey(const std::string& group) {
-  std::string key;
-  key.reserve(kGroupsTable.size() + group.size());
-  key.append(kGroupsTable.data(), kGroupsTable.size()).append(group);
-  return key;
-}
-
-std::string MetaKeyUtils::groupVal(const std::vector<std::string>& zones) {
-  return folly::join(",", zones);
-}
-
-const std::string& MetaKeyUtils::groupPrefix() { return kGroupsTable; }
-
-std::string MetaKeyUtils::parseGroupName(folly::StringPiece rawData) {
-  return rawData.subpiece(kGroupsTable.size(), rawData.size()).toString();
-}
-
-std::vector<std::string> MetaKeyUtils::parseZoneNames(folly::StringPiece rawData) {
-  std::vector<std::string> zones;
-  folly::split(',', rawData.str(), zones);
-  return zones;
-}
-
 std::string MetaKeyUtils::zoneKey(const std::string& zone) {
   std::string key;
   key.reserve(kZonesTable.size() + zone.size());

src/common/utils/MetaKeyUtils.h

@@ -110,6 +110,12 @@ class MetaKeyUtils final {
 
   static std::vector<HostAddr> parsePartValV2(folly::StringPiece val);
 
+  static std::string machineKey(std::string ip, Port port);
+
+  static const std::string& machinePrefix();
+
+  static HostAddr parseMachineKey(folly::StringPiece key);
+
   static std::string hostKey(std::string ip, Port port);
 
   static std::string hostKeyV2(std::string addr, Port port);
@@ -286,16 +292,6 @@ class MetaKeyUtils final {
   static std::tuple<BalanceTaskStatus, BalanceTaskResult, int64_t, int64_t> parseBalanceTaskVal(
       const folly::StringPiece& rawVal);
 
-  static std::string groupKey(const std::string& group);
-
-  static std::string groupVal(const std::vector<std::string>& zones);
-
-  static const std::string& groupPrefix();
-
-  static std::string parseGroupName(folly::StringPiece rawData);
-
-  static std::vector<std::string> parseZoneNames(folly::StringPiece rawData);
-
   static std::string zoneKey(const std::string& zone);
 
   static std::string zoneVal(const std::vector<HostAddr>& hosts);

src/common/utils/test/MetaKeyUtilsTest.cpp

@@ -24,8 +24,28 @@ TEST(MetaKeyUtilsTest, SpaceKeyTest) {
   spaceDesc.set_replica_factor(3);
   auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc);
   ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal));
-  ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num());
-  ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor());
+  auto properties = MetaKeyUtils::parseSpace(spaceVal);
+  ASSERT_EQ(100, properties.get_partition_num());
+  ASSERT_EQ(3, properties.get_replica_factor());
+}
+
+TEST(MetaKeyUtilsTest, SpaceKeyWithZonesTest) {
+  auto prefix = MetaKeyUtils::spacePrefix();
+  ASSERT_EQ("__spaces__", prefix);
+  auto spaceKey = MetaKeyUtils::spaceKey(101);
+  ASSERT_EQ(101, MetaKeyUtils::spaceId(spaceKey));
+  meta::cpp2::SpaceDesc spaceDesc;
+  spaceDesc.set_space_name("default");
+  spaceDesc.set_partition_num(100);
+  spaceDesc.set_replica_factor(3);
+  std::vector<std::string> zoneNames{"z1", "z2", "z3"};
+  spaceDesc.set_zone_names(std::move(zoneNames));
+  auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc);
+  ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal));
+  auto properties = MetaKeyUtils::parseSpace(spaceVal);
+  ASSERT_EQ(100, properties.get_partition_num());
+  ASSERT_EQ(3, properties.get_replica_factor());
+  ASSERT_EQ(3, properties.get_zone_names().size());
 }
 
 TEST(MetaKeyUtilsTest, PartKeyTest) {
@@ -75,6 +95,13 @@ TEST(MetaKeyUtilsTest, storeStrIpCodecTest) {
     }
   }
 
+  {
+    // kMachinesTable : key
+    auto key = MetaKeyUtils::machineKey(hostnames[0], ports[0]);
+    auto host = MetaKeyUtils::parseMachineKey(key);
+    ASSERT_EQ(host.host, hostnames[0]);
+    ASSERT_EQ(host.port, ports[0]);
+  }
   {
     // kHostsTable : key
     auto key = MetaKeyUtils::hostKey(hostnames[0], ports[0]);
@@ -183,15 +210,6 @@ TEST(MetaKeyUtilsTest, TagTest) {
   ASSERT_EQ(parsedSchema, schema);
 }
 
-TEST(MetaKeyUtilsTest, GroupTest) {
-  auto groupKey = MetaKeyUtils::groupKey("test_group");
-  ASSERT_EQ("test_group", MetaKeyUtils::parseGroupName(groupKey));
-
-  std::vector<std::string> zones = {"zone_0", "zone_1", "zone_2"};
-  auto groupValue = MetaKeyUtils::groupVal(zones);
-  ASSERT_EQ(zones, MetaKeyUtils::parseZoneNames(groupValue));
-}
-
 TEST(MetaKeyUtilsTest, ZoneTest) {
   auto zoneKey = MetaKeyUtils::zoneKey("test_zone");
   ASSERT_EQ("test_zone", MetaKeyUtils::parseZoneName(zoneKey));

src/graph/executor/CMakeLists.txt

@@ -44,6 +44,8 @@ nebula_add_library(
     algo/ProduceAllPathsExecutor.cpp
     algo/CartesianProductExecutor.cpp
     algo/SubgraphExecutor.cpp
+    admin/AddHostsExecutor.cpp
+    admin/DropHostsExecutor.cpp
     admin/SwitchSpaceExecutor.cpp
     admin/CreateUserExecutor.cpp
     admin/DropUserExecutor.cpp

src/graph/executor/Executor.cpp

@@ -16,12 +16,14 @@
 #include "graph/context/ExecutionContext.h"
 #include "graph/context/QueryContext.h"
 #include "graph/executor/ExecutionError.h"
+#include "graph/executor/admin/AddHostsExecutor.h"
 #include "graph/executor/admin/ChangePasswordExecutor.h"
 #include "graph/executor/admin/CharsetExecutor.h"
 #include "graph/executor/admin/ConfigExecutor.h"
 #include "graph/executor/admin/CreateUserExecutor.h"
 #include "graph/executor/admin/DescribeUserExecutor.h"
 #include "graph/executor/admin/DownloadExecutor.h"
+#include "graph/executor/admin/DropHostsExecutor.h"
 #include "graph/executor/admin/DropUserExecutor.h"
 #include "graph/executor/admin/GrantRoleExecutor.h"
 #include "graph/executor/admin/IngestExecutor.h"
@@ -442,20 +444,29 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) {
     case PlanNode::Kind::kSubgraph: {
       return pool->add(new SubgraphExecutor(node, qctx));
     }
-    case PlanNode::Kind::kAddZone: {
-      return pool->add(new AddZoneExecutor(node, qctx));
+    case PlanNode::Kind::kAddHosts: {
+      return pool->add(new AddHostsExecutor(node, qctx));
+    }
+    case PlanNode::Kind::kDropHosts: {
+      return pool->add(new DropHostsExecutor(node, qctx));
+    }
+    case PlanNode::Kind::kMergeZone: {
+      return pool->add(new MergeZoneExecutor(node, qctx));
+    }
+    case PlanNode::Kind::kRenameZone: {
+      return pool->add(new RenameZoneExecutor(node, qctx));
     }
     case PlanNode::Kind::kDropZone: {
       return pool->add(new DropZoneExecutor(node, qctx));
     }
+    case PlanNode::Kind::kSplitZone: {
+      return pool->add(new SplitZoneExecutor(node, qctx));
+    }
     case PlanNode::Kind::kDescribeZone: {
       return pool->add(new DescribeZoneExecutor(node, qctx));
     }
-    case PlanNode::Kind::kAddHostIntoZone: {
-      return pool->add(new AddHostIntoZoneExecutor(node, qctx));
-    }
-    case PlanNode::Kind::kDropHostFromZone: {
-      return pool->add(new DropHostFromZoneExecutor(node, qctx));
+    case PlanNode::Kind::kAddHostsIntoZone: {
+      return pool->add(new AddHostsIntoZoneExecutor(node, qctx));
     }
     case PlanNode::Kind::kShowZones: {
       return pool->add(new ListZonesExecutor(node, qctx));