Cherry pick v3.1.0 (0406-0407) (#4121)

* fix apply outdate membership change (#4107) * fix apply outdate membership change * fix transfer leader to '':0 cause crash Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * add code for part peers backward compatible (#4101) * add code for part peers backward compatible * add balance keys * fix bug: pass tests * add gflag && ignore * change int to int64_t * chage to emplace Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * fixed (#4116) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * move KW_CLEAR to unresolved keyword (#4118) * Limits for add and recover balance data or zone balance jobs (#4104) * If there are failed or stopped data balance or zone balance job, must firstly recover it * address wenhaocs's comments * address comments * add more comments * recover job * add more ut * adjust comment format * adjust comment format * adjust comment format Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> Co-authored-by: pengwei.song <90180021+pengweisong@users.noreply.github.com> Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Co-authored-by: haifei.zhao <32253291+zhaohaifei@users.noreply.github.com>
vesoft-inc · Apr 7, 2022 · 6db2cb6 · 6db2cb6
1 parent 3cee437
commit 6db2cb6
Show file tree

Hide file tree

Showing 73 changed files with 620 additions and 121 deletions.
diff --git a/.linters/cpp/checkKeyword.py b/.linters/cpp/checkKeyword.py
@@ -75,7 +75,6 @@
     'KW_ADD',
     'KW_CREATE',
     'KW_DROP',
-    'KW_CLEAR',
     'KW_REMOVE',
     'KW_IF',
     'KW_NOT',

diff --git a/scripts/nebula.service b/scripts/nebula.service
@@ -234,7 +234,7 @@ function status_daemon {
             port=${GREEN}${port}${NC}
         else
             port=${BLINK}${RED}${port}${NC}
-            if [[$daemon_name == nebula-storaged]]; then
+            if [[ $daemon_name == nebula-storaged ]]; then
                 WARN "${daemon_name} after v3.0.0 will not start service until it is added to cluster."
                 WARN "See Manage Storage hosts:${RED}ADD HOSTS${NC} in https://docs.nebula-graph.io/"
             fi

diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -915,6 +915,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("No valid job!");
     case nebula::cpp2::ErrorCode::E_JOB_NOT_IN_SPACE:
       return Status::Error("Job not existed in chosen space!");
+    case nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER:
+      return Status::Error("Need to recover failed data balance job or zone balance job firstly!");
     case nebula::cpp2::ErrorCode::E_BACKUP_EMPTY_TABLE:
       return Status::Error("Backup empty table!");
     case nebula::cpp2::ErrorCode::E_BACKUP_TABLE_FAILED:

diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h
@@ -106,6 +106,8 @@
   X(E_BALANCER_FAILURE, -2047)                                                \
   X(E_JOB_NOT_FINISHED, -2048)                                                \
   X(E_TASK_REPORT_OUT_DATE, -2049)                                            \
+  X(E_JOB_NOT_IN_SPACE, -2050)                                                \
+  X(E_JOB_NEED_RECOVER, -2051)                                                \
   X(E_INVALID_JOB, -2065)                                                     \
                                                                               \
   /* Backup Failure */                                                        \

diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp
@@ -119,6 +119,17 @@ std::string NebulaKeyUtils::systemPartKey(PartitionID partId) {
   return key;
 }
 
+// static
+std::string NebulaKeyUtils::systemBalanceKey(PartitionID partId) {
+  uint32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kSystem);
+  uint32_t type = static_cast<uint32_t>(NebulaSystemKeyType::kSystemBalance);
+  std::string key;
+  key.reserve(kSystemLen);
+  key.append(reinterpret_cast<const char*>(&item), sizeof(PartitionID))
+      .append(reinterpret_cast<const char*>(&type), sizeof(NebulaSystemKeyType));
+  return key;
+}
+
 // static
 std::string NebulaKeyUtils::kvKey(PartitionID partId, const folly::StringPiece& name) {
   std::string key;

diff --git a/src/common/utils/NebulaKeyUtils.h b/src/common/utils/NebulaKeyUtils.h
@@ -77,6 +77,8 @@ class NebulaKeyUtils final {
 
   static std::string systemPartKey(PartitionID partId);
 
+  static std::string systemBalanceKey(PartitionID partId);
+
   static std::string kvKey(PartitionID partId, const folly::StringPiece& name);
   static std::string kvPrefix(PartitionID partId);
 
@@ -189,6 +191,19 @@ class NebulaKeyUtils final {
     return static_cast<NebulaSystemKeyType>(type) == NebulaSystemKeyType::kSystemPart;
   }
 
+  static bool isSystemBalance(const folly::StringPiece& rawKey) {
+    if (rawKey.size() != kSystemLen) {
+      return false;
+    }
+    if (!isSystem(rawKey)) {
+      return false;
+    }
+    auto position = rawKey.data() + sizeof(PartitionID);
+    auto len = sizeof(NebulaSystemKeyType);
+    auto type = readInt<uint32_t>(position, len);
+    return static_cast<NebulaSystemKeyType>(type) == NebulaSystemKeyType::kSystemBalance;
+  }
+
   static VertexIDSlice getSrcId(size_t vIdLen, const folly::StringPiece& rawKey) {
     if (rawKey.size() < kEdgeLen + (vIdLen << 1)) {
       dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen);

diff --git a/src/common/utils/Types.h b/src/common/utils/Types.h
@@ -26,6 +26,7 @@ enum class NebulaKeyType : uint32_t {
 enum class NebulaSystemKeyType : uint32_t {
   kSystemCommit = 0x00000001,
   kSystemPart = 0x00000002,
+  kSystemBalance = 0x00000003,
 };
 
 enum class NebulaOperationType : uint32_t {

diff --git a/src/graph/executor/StorageAccessExecutor.cpp b/src/graph/executor/StorageAccessExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/StorageAccessExecutor.h"
 
 #include <folly/Format.h>
+
 #include "graph/context/Iterator.h"
 #include "graph/context/QueryExpressionContext.h"
 #include "graph/util/SchemaUtil.h"

diff --git a/src/graph/executor/admin/ChangePasswordExecutor.cpp b/src/graph/executor/admin/ChangePasswordExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ChangePasswordExecutor.h"
 
 #include <proxygen/lib/utils/CryptUtil.h>
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/ConfigExecutor.cpp b/src/graph/executor/admin/ConfigExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ConfigExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "common/conf/Configuration.h"
 #include "graph/planner/plan/Admin.h"
 #include "graph/util/SchemaUtil.h"

diff --git a/src/graph/executor/admin/CreateUserExecutor.cpp b/src/graph/executor/admin/CreateUserExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/CreateUserExecutor.h"
 
 #include <proxygen/lib/utils/CryptUtil.h>
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/DescribeUserExecutor.cpp b/src/graph/executor/admin/DescribeUserExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/DescribeUserExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 #include "interface/gen-cpp2/meta_types.h"
 

diff --git a/src/graph/executor/admin/DropUserExecutor.cpp b/src/graph/executor/admin/DropUserExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/DropUserExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/KillQueryExecutor.cpp b/src/graph/executor/admin/KillQueryExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/KillQueryExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/ListRolesExecutor.cpp b/src/graph/executor/admin/ListRolesExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ListRolesExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 #include "graph/service/PermissionManager.h"
 

diff --git a/src/graph/executor/admin/ListUserRolesExecutor.cpp b/src/graph/executor/admin/ListUserRolesExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ListUserRolesExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/PartExecutor.cpp b/src/graph/executor/admin/PartExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/PartExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 
 using nebula::network::NetworkUtils;

diff --git a/src/graph/executor/admin/RevokeRoleExecutor.cpp b/src/graph/executor/admin/RevokeRoleExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/RevokeRoleExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 #include "graph/service/PermissionManager.h"
 #include "interface/gen-cpp2/meta_types.h"

diff --git a/src/graph/executor/admin/ShowHostsExecutor.cpp b/src/graph/executor/admin/ShowHostsExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ShowHostsExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp b/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ShowMetaLeaderExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "common/time/TimeUtils.h"
 #include "graph/planner/plan/Admin.h"
 

diff --git a/src/graph/executor/admin/ShowServiceClientsExecutor.cpp b/src/graph/executor/admin/ShowServiceClientsExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/admin/ShowServiceClientsExecutor.h"
 
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 #include "graph/service/PermissionManager.h"
 #include "interface/gen-cpp2/meta_types.h"

diff --git a/src/graph/executor/admin/ShowStatsExecutor.cpp b/src/graph/executor/admin/ShowStatsExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/ShowStatsExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 #include "graph/service/PermissionManager.h"
 #include "graph/util/SchemaUtil.h"

diff --git a/src/graph/executor/admin/SignInServiceExecutor.cpp b/src/graph/executor/admin/SignInServiceExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/SignInServiceExecutor.h"
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/admin/SnapshotExecutor.cpp b/src/graph/executor/admin/SnapshotExecutor.cpp
@@ -3,7 +3,9 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/admin/SnapshotExecutor.h"
+
 #include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "graph/planner/plan/Admin.h"
 
 namespace nebula {

diff --git a/src/graph/executor/logic/ArgumentExecutor.cpp b/src/graph/executor/logic/ArgumentExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/logic/ArgumentExecutor.h"
+
 #include "graph/planner/plan/Logic.h"
 
 namespace nebula {

diff --git a/src/graph/executor/logic/LoopExecutor.cpp b/src/graph/executor/logic/LoopExecutor.cpp
@@ -3,8 +3,8 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/logic/LoopExecutor.h"
-#include "graph/planner/plan/Logic.h"
 
+#include "graph/planner/plan/Logic.h"
 
 namespace nebula {
 namespace graph {

diff --git a/src/graph/executor/logic/SelectExecutor.cpp b/src/graph/executor/logic/SelectExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/logic/SelectExecutor.h"
+
 #include "graph/planner/plan/Logic.h"
 
 namespace nebula {

diff --git a/src/graph/executor/maintain/EdgeExecutor.cpp b/src/graph/executor/maintain/EdgeExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/maintain/EdgeExecutor.h"
+
 #include "graph/planner/plan/Maintain.h"
 #include "graph/util/SchemaUtil.h"
 

diff --git a/src/graph/executor/maintain/FTIndexExecutor.cpp b/src/graph/executor/maintain/FTIndexExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/maintain/FTIndexExecutor.h"
+
 #include "graph/planner/plan/Maintain.h"
 #include "graph/util/FTIndexUtils.h"
 #include "interface/gen-cpp2/meta_types.h"

diff --git a/src/graph/executor/maintain/TagExecutor.cpp b/src/graph/executor/maintain/TagExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/maintain/TagExecutor.h"
+
 #include "graph/planner/plan/Maintain.h"
 #include "graph/util/SchemaUtil.h"
 

diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/mutate/DeleteExecutor.h"
+
 #include "graph/planner/plan/Mutate.h"
 #include "graph/service/GraphFlags.h"
 #include "graph/util/SchemaUtil.h"

diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/mutate/InsertExecutor.h"
+
 #include "graph/planner/plan/Mutate.h"
 #include "graph/service/GraphFlags.h"
 

diff --git a/src/graph/executor/query/AggregateExecutor.cpp b/src/graph/executor/query/AggregateExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/AggregateExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/AssignExecutor.cpp b/src/graph/executor/query/AssignExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/AssignExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/DataCollectExecutor.cpp b/src/graph/executor/query/DataCollectExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/DataCollectExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/FilterExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/GetEdgesExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 using nebula::storage::StorageClient;

diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/GetNeighborsExecutor.h"
+
 #include "graph/service/GraphFlags.h"
 
 using nebula::storage::StorageClient;

diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/IndexScanExecutor.h"
+
 #include "graph/service/GraphFlags.h"
 
 using nebula::storage::StorageClient;

diff --git a/src/graph/executor/query/InnerJoinExecutor.cpp b/src/graph/executor/query/InnerJoinExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/InnerJoinExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/IntersectExecutor.cpp b/src/graph/executor/query/IntersectExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/IntersectExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/JoinExecutor.cpp b/src/graph/executor/query/JoinExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/JoinExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/LeftJoinExecutor.cpp b/src/graph/executor/query/LeftJoinExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/LeftJoinExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/LimitExecutor.cpp b/src/graph/executor/query/LimitExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/LimitExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {

diff --git a/src/graph/executor/query/MinusExecutor.cpp b/src/graph/executor/query/MinusExecutor.cpp
@@ -3,6 +3,7 @@
 // This source code is licensed under Apache 2.0 License.
 
 #include "graph/executor/query/MinusExecutor.h"
+
 #include "graph/planner/plan/Query.h"
 
 namespace nebula {