Merge branch 'master' into hot-fix

vesoft-inc · Jan 26, 2022 · 8a985d9 · 8a985d9
2 parents 8795944 + 93693d9
commit 8a985d9
Show file tree

Hide file tree

Showing 47 changed files with 477 additions and 1,246 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -199,3 +199,92 @@ jobs:
         with:
           name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
           path:  ./build/server_*/logs/
+
+  standalone:
+    name: standalone-build
+    needs: lint
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos7
+        compiler:
+          - gcc-9.3
+    env:
+      CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      CCACHE_MAXSIZE: 8G
+    container:
+      image: vesoft/nebula-dev:${{ matrix.os }}
+      volumes:
+        - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      options: --cap-add=SYS_PTRACE
+    steps:
+      - uses: webiny/action-post-run@2.0.1
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - name: Prepare environment
+        id: prepare
+        run: |
+          [ -d build/ ] && rm -rf build/* || mkdir -p build
+          make init -C tests
+      - name: CMake
+        id: cmake
+        run: |
+          case ${{ matrix.compiler }} in
+          gcc-*)
+            case ${{ matrix.os }} in
+            centos7)
+              # build with Release type
+              cmake \
+                  -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \
+                  -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DENABLE_TESTING=on \
+                  -DENABLE_STANDALONE_VERSION=on \
+                  -GNinja \
+                  -B build
+              echo "::set-output name=j::10"
+              ;;
+            esac
+            ;;
+          esac
+      - name: Make
+        run: |
+          ccache -z
+          ninja -j $(nproc)
+          ccache -s
+        working-directory: build/
+      - name: CTest
+        env:
+          ASAN_OPTIONS: fast_unwind_on_malloc=1
+        run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure
+        working-directory: build/
+        timeout-minutes: 20
+      - name: Setup Cluster
+        run: |
+          make standalone-up
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: TCK
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} standalone-tck
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: LDBC
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: Down cluster
+        run: |
+          make RM_DIR=false down
+        working-directory: tests/
+        timeout-minutes: 2
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
+          path:  ./build/server_*/logs/
diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h
@@ -240,16 +240,18 @@ void StorageClientBase<ClientType, ClientManagerType>::getResponseImpl(
     DCHECK(!!ioThreadPool_);
     evb = ioThreadPool_->getEventBase();
   }
+  auto reqPtr = std::make_shared<std::pair<HostAddr, Request>>(std::move(request.first),
+                                                               std::move(request.second));
   folly::via(
       evb,
-      [evb, request = std::move(request), remoteFunc = std::move(remoteFunc), pro, this]() mutable {
-        auto host = request.first;
+      [evb, request = std::move(reqPtr), remoteFunc = std::move(remoteFunc), pro, this]() mutable {
+        auto host = request->first;
         auto client = clientsMan_->client(host, evb, false, FLAGS_storage_client_timeout_ms);
-        auto spaceId = request.second.get_space_id();
-        auto partsId = getReqPartsId(request.second);
-        remoteFunc(client.get(), request.second)
+        auto spaceId = request->second.get_space_id();
+        auto partsId = getReqPartsId(request->second);
+        remoteFunc(client.get(), request->second)
             .via(evb)
-            .thenValue([spaceId, pro, this](Response&& resp) mutable {
+            .thenValue([spaceId, pro, request, this](Response&& resp) mutable {
               auto& result = resp.get_result();
               for (auto& code : result.get_failed_parts()) {
                 VLOG(3) << "Failure! Failed part " << code.get_part_id() << ", failed code "

diff --git a/src/common/meta/Common.h b/src/common/meta/Common.h
@@ -62,7 +62,6 @@ using RemoteListeners =
     std::unordered_map<GraphSpaceID,
                        std::unordered_map<PartitionID, std::vector<RemoteListenerInfo>>>;
 
-
 }  // namespace meta
 }  // namespace nebula
 

diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt
@@ -101,7 +101,6 @@ nebula_add_executable(
         $<TARGET_OBJECTS:meta_version_man_obj>
         $<TARGET_OBJECTS:meta_data_upgrade_obj>
         $<TARGET_OBJECTS:meta_v2_thrift_obj>
-        $<TARGET_OBJECTS:meta_v1_thrift_obj>
         ${common_deps}
         ${storage_meta_deps}
     LIBRARIES
@@ -243,7 +242,7 @@ nebula_add_executable(
         $<TARGET_OBJECTS:meta_http_handler>
         $<TARGET_OBJECTS:meta_version_man_obj>
         $<TARGET_OBJECTS:meta_data_upgrade_obj>
-        $<TARGET_OBJECTS:meta_v1_thrift_obj>
+        $<TARGET_OBJECTS:meta_v2_thrift_obj>
         ${storage_meta_deps}
         ${common_deps}
     LIBRARIES

diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp
@@ -145,13 +145,8 @@ std::unique_ptr<nebula::kvstore::KVStore> initKV(std::vector<nebula::HostAddr> p
     LOG(ERROR) << "Meta version is invalid";
     return nullptr;
   } else if (version == nebula::meta::MetaVersion::V1) {
-    auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(engine);
-    if (!ret.ok()) {
-      LOG(ERROR) << "Update meta from V1 to V2 failed " << ret;
-      return nullptr;
-    }
-
-    nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V2);
+    LOG(ERROR) << "Can't upgrade meta from V1 to V3";
+    return nullptr;
   } else if (version == nebula::meta::MetaVersion::V2) {
     auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine);
     if (!ret.ok()) {

diff --git a/src/daemons/StandAloneDaemon.cpp b/src/daemons/StandAloneDaemon.cpp
@@ -163,6 +163,13 @@ int main(int argc, char *argv[]) {
     return EXIT_FAILURE;
   }
 
+  // load the time zone data
+  status = nebula::time::Timezone::init();
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return EXIT_FAILURE;
+  }
+
   // Initialize the global timezone, it's only used for datetime type compute
   // won't affect the process timezone.
   status = nebula::time::Timezone::initializeGlobalTimezone();

diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp
@@ -80,11 +80,10 @@ folly::Future<Status> TraverseExecutor::traverse() {
     DataSet emptyResult;
     return finish(ResultBuilder().value(Value(std::move(emptyResult))).build());
   }
-  getNeighbors();
-  return promise_.getFuture();
+  return getNeighbors();
 }
 
-void TraverseExecutor::getNeighbors() {
+folly::Future<Status> TraverseExecutor::getNeighbors() {
   currentStep_++;
   time::Duration getNbrTime;
   StorageClient* storageClient = qctx_->getStorageClient();
@@ -93,7 +92,7 @@ void TraverseExecutor::getNeighbors() {
                                           qctx()->rctx()->session()->id(),
                                           qctx()->plan()->id(),
                                           qctx()->plan()->isProfileEnabled());
-  storageClient
+  return storageClient
       ->getNeighbors(param,
                      reqDs_.colNames,
                      std::move(reqDs_.rows),
@@ -112,7 +111,7 @@ void TraverseExecutor::getNeighbors() {
       .thenValue([this, getNbrTime](StorageRpcResponse<GetNeighborsResponse>&& resp) mutable {
         SCOPED_TIMER(&execTime_);
         addStats(resp, getNbrTime.elapsedInUSec());
-        handleResponse(resp);
+        return handleResponse(std::move(resp));
       });
 }
 
@@ -140,11 +139,11 @@ void TraverseExecutor::addStats(RpcResponse& resp, int64_t getNbrTimeInUSec) {
   otherStats_.emplace(folly::sformat("step {}", currentStep_), ss.str());
 }
 
-void TraverseExecutor::handleResponse(RpcResponse& resps) {
+folly::Future<Status> TraverseExecutor::handleResponse(RpcResponse&& resps) {
   SCOPED_TIMER(&execTime_);
   auto result = handleCompleteness(resps, FLAGS_accept_partial_success);
   if (!result.ok()) {
-    promise_.setValue(std::move(result).status());
+    return folly::makeFuture<Status>(std::move(result).status());
   }
 
   auto& responses = resps.responses();
@@ -162,21 +161,20 @@ void TraverseExecutor::handleResponse(RpcResponse& resps) {
 
   auto status = buildInterimPath(iter.get());
   if (!status.ok()) {
-    promise_.setValue(status);
-    return;
+    return folly::makeFuture<Status>(std::move(status));
   }
   if (!isFinalStep()) {
     if (reqDs_.rows.empty()) {
       if (range_ != nullptr) {
-        promise_.setValue(buildResult());
+        return folly::makeFuture<Status>(buildResult());
       } else {
-        promise_.setValue(Status::OK());
+        return folly::makeFuture<Status>(Status::OK());
       }
     } else {
-      getNeighbors();
+      return getNeighbors();
     }
   } else {
-    promise_.setValue(buildResult());
+    return folly::makeFuture<Status>(buildResult());
   }
 }
 

diff --git a/src/graph/executor/query/TraverseExecutor.h b/src/graph/executor/query/TraverseExecutor.h
@@ -41,9 +41,9 @@ class TraverseExecutor final : public StorageAccessExecutor {
 
   void addStats(RpcResponse& resps, int64_t getNbrTimeInUSec);
 
-  void getNeighbors();
+  folly::Future<Status> getNeighbors();
 
-  void handleResponse(RpcResponse& resps);
+  folly::Future<Status> handleResponse(RpcResponse&& resps);
 
   Status buildInterimPath(GetNeighborsIter* iter);
 
@@ -74,7 +74,6 @@ class TraverseExecutor final : public StorageAccessExecutor {
  private:
   DataSet reqDs_;
   const Traverse* traverse_{nullptr};
-  folly::Promise<Status> promise_;
   MatchStepRange* range_{nullptr};
   size_t currentStep_{0};
   std::list<std::unordered_map<Value, Paths>> paths_;

diff --git a/src/graph/executor/test/StorageServerStub.cpp b/src/graph/executor/test/StorageServerStub.cpp
@@ -23,7 +23,6 @@ std::shared_ptr<GraphStorageLocalServer> instance_ = nullptr;
 
 void GraphStorageLocalServer::setThreadManager(
     std::shared_ptr<apache::thrift::concurrency::ThreadManager> threadManager) {
-  // lock?
   threadManager_ = threadManager;
 }
 

diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp
@@ -84,3 +84,4 @@ static bool ValidateSessIdleTimeout(const char* flagname, int32_t value) {
   return false;
 }
 DEFINE_validator(session_idle_timeout_secs, &ValidateSessIdleTimeout);
+DEFINE_validator(client_idle_timeout_secs, &ValidateSessIdleTimeout);
diff --git a/src/graph/validator/AdminValidator.cpp b/src/graph/validator/AdminValidator.cpp
@@ -21,8 +21,9 @@ Status CreateSpaceValidator::validateImpl() {
   auto status = Status::OK();
   spaceDesc_.space_name_ref() = std::move(*(sentence->spaceName()));
   if (sentence->zoneNames()) {
-    spaceDesc_.zone_names_ref() = sentence->zoneNames()->zoneNames();
+    return Status::SemanticError("Create space with zone is unsupported");
   }
+
   StatusOr<std::string> retStatusOr;
   std::string result;
   auto *charsetInfo = qctx_->getCharsetInfo();
@@ -323,10 +324,6 @@ Status ShowListenerValidator::toPlan() {
 }
 
 Status AddHostsValidator::validateImpl() {
-  return Status::OK();
-}
-
-Status AddHostsValidator::toPlan() {
   auto sentence = static_cast<AddHostsSentence *>(sentence_);
   auto hosts = sentence->hosts()->hosts();
   if (hosts.empty()) {
@@ -337,18 +334,19 @@ Status AddHostsValidator::toPlan() {
   if (it != hosts.end()) {
     return Status::SemanticError("Host have duplicated");
   }
+  return Status::OK();
+}
 
+Status AddHostsValidator::toPlan() {
+  auto sentence = static_cast<AddHostsSentence *>(sentence_);
+  auto hosts = sentence->hosts()->hosts();
   auto *addHost = AddHosts::make(qctx_, nullptr, hosts);
   root_ = addHost;
   tail_ = root_;
   return Status::OK();
 }
 
 Status DropHostsValidator::validateImpl() {
-  return Status::OK();
-}
-
-Status DropHostsValidator::toPlan() {
   auto sentence = static_cast<DropHostsSentence *>(sentence_);
   auto hosts = sentence->hosts()->hosts();
   if (hosts.empty()) {
@@ -359,7 +357,12 @@ Status DropHostsValidator::toPlan() {
   if (it != hosts.end()) {
     return Status::SemanticError("Host have duplicated");
   }
+  return Status::OK();
+}
 
+Status DropHostsValidator::toPlan() {
+  auto sentence = static_cast<DropHostsSentence *>(sentence_);
+  auto hosts = sentence->hosts()->hosts();
   auto *dropHost = DropHosts::make(qctx_, nullptr, hosts);
   root_ = dropHost;
   tail_ = root_;

diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp
@@ -496,7 +496,7 @@ Status ShowEdgeIndexStatusValidator::toPlan() {
 }
 
 Status MergeZoneValidator::validateImpl() {
-  return Status::OK();
+  return Status::SemanticError("Merge zone is unsupported");
 }
 
 Status MergeZoneValidator::toPlan() {
@@ -509,7 +509,7 @@ Status MergeZoneValidator::toPlan() {
 }
 
 Status RenameZoneValidator::validateImpl() {
-  return Status::OK();
+  return Status::SemanticError("Rename zone is unsupported");
 }
 
 Status RenameZoneValidator::toPlan() {
@@ -534,7 +534,7 @@ Status DropZoneValidator::toPlan() {
 }
 
 Status DivideZoneValidator::validateImpl() {
-  return Status::OK();
+  return Status::SemanticError("Divide zone is unsupported");
 }
 
 Status DivideZoneValidator::toPlan() {
@@ -558,11 +558,11 @@ Status DescribeZoneValidator::toPlan() {
   return Status::OK();
 }
 
-Status ListZonesValidator::validateImpl() {
-  return Status::OK();
+Status ShowZonesValidator::validateImpl() {
+  return Status::SemanticError("Show zones is unsupported");
 }
 
-Status ListZonesValidator::toPlan() {
+Status ShowZonesValidator::toPlan() {
   auto *doNode = ListZones::make(qctx_, nullptr);
   root_ = doNode;
   tail_ = root_;

diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h
@@ -382,9 +382,9 @@ class DescribeZoneValidator final : public Validator {
   Status toPlan() override;
 };
 
-class ListZonesValidator final : public Validator {
+class ShowZonesValidator final : public Validator {
  public:
-  ListZonesValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {
+  ShowZonesValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {
     setNoSpaceRequired();
   }