Merge branch 'master' into test/enable-skip-tests

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: I want to ask a question.
+labels: question
+---
+
+## General Question
+
+<!--
+
+Before asking a question, make sure you have:
+
+- Searched existing questions [forum-EN](https://discuss.nebula-graph.io/) or [forum-CH](https://discuss.nebula-graph.com.cn/).
+- Googled your question.
+- Searched open and closed [GitHub issues](https://github.com/vesoft-inc/nebula/issues?q=is%3Aissue).
+- Read the [documentation-EN](https://docs.nebula-graph.io/) or [documentation-CH](https://docs.nebula-graph.com.cn/).
+
+-->

.linters/cpp/checkKeyword.py

@@ -20,6 +20,8 @@
     'KW_XOR',
     'KW_USE',
     'KW_SET',
+    'KW_LIST',
+    'KW_MAP',
     'KW_FROM',
     'KW_WHERE',
     'KW_MATCH',

README.md

@@ -58,7 +58,7 @@ Please note that you need to install **Nebula Graph**, either by [installing sou
 ## Getting help
 In case you encounter any problems playing around **Nebula Graph**, please reach out for help:
 * [FAQ](https://docs.nebula-graph.io/2.0/2.quick-start/0.FAQ/)
-* [Forum](https://discuss.nebula-graph.io/)
+* [Discussions](https://github.com/vesoft-inc/nebula/discussions)
 
 ## Documentation
 

conf/nebula-graphd.conf.default

@@ -51,10 +51,11 @@
 --reuse_port=false
 # Backlog of the listen socket, adjust this together with net.core.somaxconn
 --listen_backlog=1024
-# Seconds before the idle connections are closed, 0 for never closed
---client_idle_timeout_secs=0
-# Seconds before the idle sessions are expired, 0 for no expiration
---session_idle_timeout_secs=0
+# The number of seconds Nebula service waits before closing the idle connections
+--client_idle_timeout_secs=28800
+# The number of seconds before idle sessions expire
+# The range should be in [1, 604800]
+--session_idle_timeout_secs=28800
 # The number of threads to accept incoming connections
 --num_accept_threads=1
 # The number of networking IO threads, 0 for # of CPU cores

conf/nebula-graphd.conf.production

@@ -49,10 +49,11 @@
 --reuse_port=false
 # Backlog of the listen socket, adjust this together with net.core.somaxconn
 --listen_backlog=1024
-# Seconds before the idle connections are closed, 0 for never closed
---client_idle_timeout_secs=0
-# Seconds before the idle sessions are expired, 0 for no expiration
---session_idle_timeout_secs=60000
+# The number of seconds Nebula service waits before closing the idle connections
+--client_idle_timeout_secs=28800
+# The number of seconds before idle sessions expire
+# The range should be in [1, 604800]
+--session_idle_timeout_secs=28800
 # The number of threads to accept incoming connections
 --num_accept_threads=1
 # The number of networking IO threads, 0 for # of CPU cores

src/clients/meta/MetaClient.cpp

@@ -266,7 +266,7 @@ bool MetaClient::loadData() {
   }
 
   auto hostsRet = listHosts().get();
-  if (!ret.ok()) {
+  if (!hostsRet.ok()) {
     LOG(ERROR) << "List hosts failed, status:" << hostsRet.status();
     return false;
   }
@@ -2414,6 +2414,21 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
     } else {
       req.set_leader_partIds(std::move(leaderIds));
     }
+
+    kvstore::SpaceDiskPartsMap diskParts;
+    if (listener_ != nullptr) {
+      listener_->fetchDiskParts(diskParts);
+      if (diskParts_ != diskParts) {
+        {
+          folly::RWSpinLock::WriteHolder holder(&diskPartsLock_);
+          diskParts_.clear();
+          diskParts_ = diskParts;
+        }
+        req.set_disk_parts(diskParts);
+      }
+    } else {
+      req.set_disk_parts(diskParts);
+    }
   }
 
   folly::Promise<StatusOr<bool>> promise;
@@ -3116,102 +3131,6 @@ folly::Future<StatusOr<std::vector<cpp2::Zone>>> MetaClient::listZones() {
   return future;
 }
 
-folly::Future<StatusOr<bool>> MetaClient::addGroup(std::string groupName,
-                                                   std::vector<std::string> zoneNames) {
-  cpp2::AddGroupReq req;
-  req.set_group_name(std::move(groupName));
-  req.set_zone_names(std::move(zoneNames));
-
-  folly::Promise<StatusOr<bool>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_addGroup(request); },
-      [](cpp2::ExecResp&& resp) -> bool {
-        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
-      },
-      std::move(promise));
-  return future;
-}
-
-folly::Future<StatusOr<bool>> MetaClient::dropGroup(std::string groupName) {
-  cpp2::DropGroupReq req;
-  req.set_group_name(std::move(groupName));
-
-  folly::Promise<StatusOr<bool>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_dropGroup(request); },
-      [](cpp2::ExecResp&& resp) -> bool {
-        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
-      },
-      std::move(promise));
-  return future;
-}
-
-folly::Future<StatusOr<bool>> MetaClient::addZoneIntoGroup(std::string zoneName,
-                                                           std::string groupName) {
-  cpp2::AddZoneIntoGroupReq req;
-  req.set_zone_name(zoneName);
-  req.set_group_name(groupName);
-
-  folly::Promise<StatusOr<bool>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_addZoneIntoGroup(request); },
-      [](cpp2::ExecResp&& resp) -> bool {
-        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
-      },
-      std::move(promise));
-  return future;
-}
-
-folly::Future<StatusOr<bool>> MetaClient::dropZoneFromGroup(std::string zoneName,
-                                                            std::string groupName) {
-  cpp2::DropZoneFromGroupReq req;
-  req.set_zone_name(zoneName);
-  req.set_group_name(groupName);
-
-  folly::Promise<StatusOr<bool>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_dropZoneFromGroup(request); },
-      [](cpp2::ExecResp&& resp) -> bool {
-        return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
-      },
-      std::move(promise));
-  return future;
-}
-
-folly::Future<StatusOr<std::vector<std::string>>> MetaClient::getGroup(std::string groupName) {
-  cpp2::GetGroupReq req;
-  req.set_group_name(std::move(groupName));
-
-  folly::Promise<StatusOr<std::vector<std::string>>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_getGroup(request); },
-      [](cpp2::GetGroupResp&& resp) -> decltype(auto) { return resp.get_zone_names(); },
-      std::move(promise));
-  return future;
-}
-
-folly::Future<StatusOr<std::vector<cpp2::Group>>> MetaClient::listGroups() {
-  cpp2::ListGroupsReq req;
-  folly::Promise<StatusOr<std::vector<cpp2::Group>>> promise;
-  auto future = promise.getFuture();
-  getResponse(
-      std::move(req),
-      [](auto client, auto request) { return client->future_listGroups(request); },
-      [](cpp2::ListGroupsResp&& resp) -> decltype(auto) { return resp.get_groups(); },
-      std::move(promise));
-  return future;
-}
-
 folly::Future<StatusOr<cpp2::StatsItem>> MetaClient::getStats(GraphSpaceID spaceId) {
   cpp2::GetStatsReq req;
   req.set_space_id(spaceId);

src/clients/meta/MetaClient.h

@@ -27,6 +27,7 @@
 #include "interface/gen-cpp2/MetaServiceAsyncClient.h"
 #include "interface/gen-cpp2/common_types.h"
 #include "interface/gen-cpp2/meta_types.h"
+#include "kvstore/DiskManager.h"
 
 DECLARE_int32(meta_client_retry_times);
 DECLARE_int32(heartbeat_interval_secs);
@@ -162,6 +163,7 @@ class MetaChangedListener {
   virtual void onPartUpdated(const PartHosts& partHosts) = 0;
   virtual void fetchLeaderInfo(
       std::unordered_map<GraphSpaceID, std::vector<cpp2::LeaderInfo>>& leaders) = 0;
+  virtual void fetchDiskParts(kvstore::SpaceDiskPartsMap& diskParts) = 0;
   virtual void onListenerAdded(GraphSpaceID spaceId,
                                PartitionID partId,
                                const ListenerHosts& listenerHosts) = 0;
@@ -607,18 +609,6 @@ class MetaClient {
 
   folly::Future<StatusOr<std::vector<cpp2::Zone>>> listZones();
 
-  folly::Future<StatusOr<bool>> addGroup(std::string groupName, std::vector<std::string> zoneNames);
-
-  folly::Future<StatusOr<bool>> dropGroup(std::string groupName);
-
-  folly::Future<StatusOr<bool>> addZoneIntoGroup(std::string zoneName, std::string groupName);
-
-  folly::Future<StatusOr<bool>> dropZoneFromGroup(std::string zoneName, std::string groupName);
-
-  folly::Future<StatusOr<std::vector<std::string>>> getGroup(std::string groupName);
-
-  folly::Future<StatusOr<std::vector<cpp2::Group>>> listGroups();
-
   Status refreshCache();
 
   folly::Future<StatusOr<cpp2::StatsItem>> getStats(GraphSpaceID spaceId);
@@ -730,9 +720,14 @@ class MetaClient {
   std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
   std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
 
+  // heartbeat is a single thread, maybe leaderIdsLock_ and diskPartsLock_ is useless?
   // leaderIdsLock_ is used to protect leaderIds_
   std::unordered_map<GraphSpaceID, std::vector<cpp2::LeaderInfo>> leaderIds_;
   folly::RWSpinLock leaderIdsLock_;
+  // diskPartsLock_ is used to protect diskParts_;
+  kvstore::SpaceDiskPartsMap diskParts_;
+  folly::RWSpinLock diskPartsLock_;
+
   std::atomic<int64_t> localDataLastUpdateTime_{-1};
   std::atomic<int64_t> localCfgLastUpdateTime_{-1};
   std::atomic<int64_t> metadLastUpdateTime_{0};

src/clients/storage/CMakeLists.txt

@@ -3,14 +3,8 @@
 # This source code is licensed under Apache 2.0 License.
 
 nebula_add_library(
-    graph_storage_client_obj OBJECT
-    GraphStorageClient.cpp
-)
-
-
-nebula_add_library(
-    general_storage_client_obj OBJECT
-    GeneralStorageClient.cpp
+    storage_client_obj OBJECT
+    StorageClient.cpp
 )