Cherry pick v3.2.0 (0616-0627) (#4354)

* force cache the docker layer (#4331) * check god role exist when meta init (#4330) * check god role exist when meta init * return error if kv fail Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> * Fix object pool mtsafe. (#4332) * Fix object pool mtsafe. * Fix lock. * Fixed web service crash (#4334) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * Fix get edges transform rule. (#4328) 1. Input/Ouput variables. 2. Keep column names of Limit same with input plan node. Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * fix rc docker (#4336) * add lock (#4352) * fix map concurrency issue (#4344) * fix mutex in map * add test * move the order Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * add stats under index conditions (#4353) Co-authored-by: Harris.Chu <1726587+HarrisChu@users.noreply.github.com> Co-authored-by: jimingquan <mingquan.ji@vesoft.com> Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> Co-authored-by: shylock <33566796+Shylock-Hg@users.noreply.github.com> Co-authored-by: dutor <440396+dutor@users.noreply.github.com> Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com>
vesoft-inc · Jun 27, 2022 · dfe737c · dfe737c
1 parent f65c5c2
commit dfe737c
Show file tree

Hide file tree

Showing 25 changed files with 159 additions and 59 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -82,6 +82,7 @@ jobs:
           tags: |
             vesoft/nebula-graphd:nightly
           target: graphd
+          cache-to: type=local,dest=/tmp/buildx-cache,mode=max
           push: true
       - uses: docker/build-push-action@v2
         with:
@@ -91,6 +92,7 @@ jobs:
           tags: |
             vesoft/nebula-storaged:nightly
           target: storaged
+          cache-from: type=local,src=/tmp/buildx-cache
           push: true
       - uses: docker/build-push-action@v2
         with:
@@ -100,6 +102,7 @@ jobs:
           tags: |
             vesoft/nebula-metad:nightly
           target: metad
+          cache-from: type=local,src=/tmp/buildx-cache
           push: true
       - uses: docker/build-push-action@v2
         with:
@@ -109,7 +112,11 @@ jobs:
           tags: |
             vesoft/nebula-tools:nightly
           target: tools
+          cache-from: type=local,src=/tmp/buildx-cache
           push: true
+      - name: delete the cache
+        run: |
+          rm -rf /tmp/buildx-cache
 
   coverage:
     name: coverage

diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
@@ -117,6 +117,7 @@ jobs:
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-graphd:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.graphdTag }}
           target: graphd
+          cache-to: type=local,dest=/tmp/buildx-cache,mode=max
           push: true
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
@@ -128,11 +129,12 @@ jobs:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
           tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.tag }}
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.storagedTag }}
           target: storaged
           push: true
+          cache-from: type=local,src=/tmp/buildx-cache
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
             VERSION=${{ steps.tagname.outputs.tagnum }}
@@ -143,11 +145,12 @@ jobs:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
           tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.tag }}
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.metadTag }}
           target: metad
           push: true
+          cache-from: type=local,src=/tmp/buildx-cache
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
             VERSION=${{ steps.tagname.outputs.tagnum }}
@@ -158,14 +161,18 @@ jobs:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
           tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.tag }}
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.toolsTag }}
           target: tools
           push: true
+          cache-from: type=local,src=/tmp/buildx-cache
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
             VERSION=${{ steps.tagname.outputs.tagnum }}
+      - name: delete the cache
+        run: |
+          rm -rf /tmp/buildx-cache
   test:
     name: test
     needs: package

diff --git a/src/common/base/ObjectPool.h b/src/common/base/ObjectPool.h
@@ -38,7 +38,9 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable {
 
   template <typename T, typename... Args>
   T *makeAndAdd(Args &&... args) {
+    lock_.lock();
     void *ptr = arena_.allocateAligned(sizeof(T));
+    lock_.unlock();
     return add(new (ptr) T(std::forward<Args>(args)...));
   }
 

diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp
@@ -176,11 +176,16 @@ int main(int argc, char* argv[]) {
     }
     if (nebula::value(ret) == localhost) {
       LOG(INFO) << "Check and init root user";
-      if (!nebula::meta::RootUserMan::isUserExists(gKVStore.get())) {
-        if (!nebula::meta::RootUserMan::initRootUser(gKVStore.get())) {
-          LOG(ERROR) << "Init root user failed";
-          return EXIT_FAILURE;
-        }
+      auto checkRet = nebula::meta::RootUserMan::isGodExists(gKVStore.get());
+      if (!nebula::ok(checkRet)) {
+        auto retCode = nebula::error(checkRet);
+        LOG(ERROR) << "Parser God Role error:" << apache::thrift::util::enumNameSafe(retCode);
+        return EXIT_FAILURE;
+      }
+      auto existGod = nebula::value(checkRet);
+      if (!existGod && !nebula::meta::RootUserMan::initRootUser(gKVStore.get())) {
+        LOG(ERROR) << "Init root user failed";
+        return EXIT_FAILURE;
       }
     }
   }

diff --git a/src/daemons/StandAloneDaemon.cpp b/src/daemons/StandAloneDaemon.cpp
@@ -234,11 +234,16 @@ int main(int argc, char *argv[]) {
       }
       if (nebula::value(ret) == metaLocalhost) {
         LOG(INFO) << "Check and init root user";
-        if (!nebula::meta::RootUserMan::isUserExists(gMetaKVStore.get())) {
-          if (!nebula::meta::RootUserMan::initRootUser(gMetaKVStore.get())) {
-            LOG(ERROR) << "Init root user failed";
-            return;
-          }
+        auto checkRet = nebula::meta::RootUserMan::isGodExists(gMetaKVStore.get());
+        if (!nebula::ok(checkRet)) {
+          auto retCode = nebula::error(checkRet);
+          LOG(ERROR) << "Parser God Role error:" << apache::thrift::util::enumNameSafe(retCode);
+          return;
+        }
+        auto existGod = nebula::value(checkRet);
+        if (!existGod && !nebula::meta::RootUserMan::initRootUser(gMetaKVStore.get())) {
+          LOG(ERROR) << "Init root user failed";
+          return;
         }
       }
     }

diff --git a/src/graph/executor/algo/BatchShortestPath.cpp b/src/graph/executor/algo/BatchShortestPath.cpp
@@ -135,7 +135,7 @@ folly::Future<Status> BatchShortestPath::getNeighbors(size_t rowNum, size_t step
                      nullptr)
       .via(qctx_->rctx()->runner())
       .thenValue([this, rowNum, reverse, stepNum, getNbrTime](auto&& resp) {
-        addStats(resp, stats_, stepNum, getNbrTime.elapsedInUSec(), reverse);
+        addStats(resp, stepNum, getNbrTime.elapsedInUSec(), reverse);
         return buildPath(rowNum, std::move(resp), reverse);
       });
 }

diff --git a/src/graph/executor/algo/ShortestPathBase.cpp b/src/graph/executor/algo/ShortestPathBase.cpp
@@ -38,7 +38,7 @@ folly::Future<std::vector<Value>> ShortestPathBase::getMeetVidsProps(
                  nullptr)
       .via(qctx_->rctx()->runner())
       .thenValue([this, getPropsTime](PropRpcResponse&& resp) {
-        addStats(resp, stats_, getPropsTime.elapsedInUSec());
+        addStats(resp, getPropsTime.elapsedInUSec());
         return handlePropResp(std::move(resp));
       });
 }
@@ -168,7 +168,6 @@ Status ShortestPathBase::handleErrorCode(nebula::cpp2::ErrorCode code, Partition
 }
 
 void ShortestPathBase::addStats(RpcResponse& resp,
-                                std::unordered_map<std::string, std::string>* stats,
                                 size_t stepNum,
                                 int64_t timeInUSec,
                                 bool reverse) const {
@@ -193,15 +192,17 @@ void ShortestPathBase::addStats(RpcResponse& resp,
   }
   ss << "\n}";
   if (reverse) {
-    stats->emplace(folly::sformat("reverse step {}", stepNum), ss.str());
+    statsLock_.lock();
+    stats_->emplace(folly::sformat("reverse step {}", stepNum), ss.str());
+    statsLock_.unlock();
   } else {
-    stats->emplace(folly::sformat("step {}", stepNum), ss.str());
+    statsLock_.lock();
+    stats_->emplace(folly::sformat("step {}", stepNum), ss.str());
+    statsLock_.unlock();
   }
 }
 
-void ShortestPathBase::addStats(PropRpcResponse& resp,
-                                std::unordered_map<std::string, std::string>* stats,
-                                int64_t timeInUSec) const {
+void ShortestPathBase::addStats(PropRpcResponse& resp, int64_t timeInUSec) const {
   auto& hostLatency = resp.hostLatency();
   std::stringstream ss;
   ss << "{\n";
@@ -217,7 +218,7 @@ void ShortestPathBase::addStats(PropRpcResponse& resp,
     ss << "\n}";
   }
   ss << "\n}";
-  stats->emplace(folly::sformat("get_prop "), ss.str());
+  stats_->emplace(folly::sformat("get_prop "), ss.str());
 }
 
 }  // namespace graph

diff --git a/src/graph/executor/algo/ShortestPathBase.h b/src/graph/executor/algo/ShortestPathBase.h
@@ -43,15 +43,9 @@ class ShortestPathBase {
 
   Status handleErrorCode(nebula::cpp2::ErrorCode code, PartitionID partId) const;
 
-  void addStats(RpcResponse& resp,
-                std::unordered_map<std::string, std::string>* stats,
-                size_t stepNum,
-                int64_t timeInUSec,
-                bool reverse) const;
+  void addStats(RpcResponse& resp, size_t stepNum, int64_t timeInUSec, bool reverse) const;
 
-  void addStats(PropRpcResponse& resp,
-                std::unordered_map<std::string, std::string>* stats,
-                int64_t timeInUSec) const;
+  void addStats(PropRpcResponse& resp, int64_t timeInUSec) const;
 
   template <typename Resp>
   StatusOr<Result::State> handleCompleteness(const storage::StorageRpcResponse<Resp>& rpcResp,
@@ -86,6 +80,7 @@ class ShortestPathBase {
   std::unordered_map<std::string, std::string>* stats_{nullptr};
   size_t maxStep_;
   bool singleShortest_{true};
+  folly::SpinLock statsLock_;
 
   std::vector<DataSet> resultDs_;
   std::vector<DataSet> leftVids_;

diff --git a/src/graph/executor/algo/SingleShortestPath.cpp b/src/graph/executor/algo/SingleShortestPath.cpp
@@ -108,7 +108,7 @@ folly::Future<Status> SingleShortestPath::getNeighbors(size_t rowNum,
                      nullptr)
       .via(qctx_->rctx()->runner())
       .thenValue([this, rowNum, stepNum, getNbrTime, reverse](auto&& resp) {
-        addStats(resp, stats_, stepNum, getNbrTime.elapsedInUSec(), reverse);
+        addStats(resp, stepNum, getNbrTime.elapsedInUSec(), reverse);
         return buildPath(rowNum, std::move(resp), reverse);
       });
 }
@@ -264,6 +264,9 @@ folly::Future<bool> SingleShortestPath::buildEvenPath(size_t rowNum,
       return false;
     }
     for (auto& meetVertex : vertices) {
+      if (!meetVertex.isVertex()) {
+        continue;
+      }
       auto meetVid = meetVertex.getVertex().vid;
       auto leftPaths = createLeftPath(rowNum, meetVid);
       auto rightPaths = createRightPath(rowNum, meetVid, false);

diff --git a/src/graph/optimizer/rule/CollapseProjectRule.cpp b/src/graph/optimizer/rule/CollapseProjectRule.cpp
@@ -75,6 +75,7 @@ StatusOr<OptRule::TransformResult> CollapseProjectRule::transform(
   // 2. find link according to propRefNames and colNames in ProjBelow
   std::unordered_map<std::string, Expression*> rewriteMap;
   auto colNames = projBelow->colNames();
+  DCHECK_EQ(colNames.size(), colsBelow.size());
   for (size_t i = 0; i < colNames.size(); ++i) {
     if (uniquePropRefNames.count(colNames[i])) {
       auto colExpr = colsBelow[i]->expr();

diff --git a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp
@@ -85,7 +85,6 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformAppendVerticesLimitRule::tra
   auto newLimit = limit->clone();
   auto newLimitGroup = OptGroup::create(ctx);
   auto newLimitGroupNode = newLimitGroup->makeGroupNode(newLimit);
-  newLimit->setOutputVar(limit->outputVar());
   newProject->setInputVar(newLimit->outputVar());
 
   newProjectGroupNode->dependsOn(newLimitGroup);
@@ -105,8 +104,8 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformAppendVerticesLimitRule::tra
   auto newAppendVertices = appendVertices->clone();
   auto newAppendVerticesGroup = OptGroup::create(ctx);
   auto colSize = appendVertices->colNames().size();
-  newAppendVertices->setOutputVar(appendVertices->outputVar());
   newLimit->setInputVar(newAppendVertices->outputVar());
+  newLimit->setColNames(newAppendVertices->colNames());  // Limit keep column names same with input
   newAppendVertices->setColNames(
       {appendVertices->colNames()[colSize - 2], appendVertices->colNames()[colSize - 1]});
   auto newAppendVerticesGroupNode = newAppendVerticesGroup->makeGroupNode(newAppendVertices);
@@ -123,19 +122,19 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformAppendVerticesLimitRule::tra
   auto *newProj =
       GetEdgesTransformUtils::projectEdges(qctx, newScanEdges, traverse->colNames().back());
   newProj->setInputVar(newScanEdges->outputVar());
-  newProj->setOutputVar(newAppendVertices->inputVar());
   newProj->setColNames({traverse->colNames().back()});
   auto newProjGroup = OptGroup::create(ctx);
   auto newProjGroupNode = newProjGroup->makeGroupNode(newProj);
 
   newAppendVerticesGroupNode->dependsOn(newProjGroup);
+  newAppendVertices->setInputVar(newProj->outputVar());
   newProjGroupNode->dependsOn(newScanEdgesGroup);
   for (auto dep : scanVerticesGroupNode->dependencies()) {
     newScanEdgesGroupNode->dependsOn(dep);
   }
 
   TransformResult result;
-  result.eraseCurr = true;
+  result.eraseAll = true;
   result.newGroupNodes.emplace_back(newProjectGroupNode);
   return result;
 }

diff --git a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp
@@ -72,8 +72,8 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformRule::transform(
   auto project = static_cast<const Project *>(projectGroupNode->node());
 
   auto newProject = project->clone();
-  auto newProjectGroupNode = OptGroupNode::create(ctx, newProject, projectGroupNode->group());
   newProject->setOutputVar(project->outputVar());
+  auto newProjectGroupNode = OptGroupNode::create(ctx, newProject, projectGroupNode->group());
 
   auto limitGroupNode = matched.dependencies.front().node;
   auto limit = static_cast<const Limit *>(limitGroupNode->node());
@@ -85,11 +85,11 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformRule::transform(
 
   auto newLimit = limit->clone();
   auto newLimitGroup = OptGroup::create(ctx);
-  newLimit->setOutputVar(limit->outputVar());
 
   auto newLimitGroupNode = newLimitGroup->makeGroupNode(newLimit);
 
   newProjectGroupNode->dependsOn(newLimitGroup);
+  newProject->setInputVar(newLimit->outputVar());
 
   auto *newScanEdges = GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->count(qctx));
   if (newScanEdges == nullptr) {
@@ -100,14 +100,15 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformRule::transform(
 
   auto *newProj =
       GetEdgesTransformUtils::projectEdges(qctx, newScanEdges, traverse->colNames().back());
-  newProj->setInputVar(newScanEdges->outputVar());
-  newProj->setOutputVar(traverse->outputVar());
   newProj->setColNames({traverse->colNames().back()});
   auto newProjGroup = OptGroup::create(ctx);
   auto newProjGroupNode = newProjGroup->makeGroupNode(newProj);
 
   newLimitGroupNode->dependsOn(newProjGroup);
+  newLimit->setInputVar(newProj->outputVar());
+  newLimit->setColNames(newProj->colNames());  // Limit keep column names same with input
   newProjGroupNode->dependsOn(newScanEdgesGroup);
+  newProj->setInputVar(newScanEdges->outputVar());
   newScanEdgesGroupNode->setDeps(scanVerticesGroupNode->dependencies());
 
   TransformResult result;

diff --git a/src/graph/optimizer/rule/GetEdgesTransformUtils.h b/src/graph/optimizer/rule/GetEdgesTransformUtils.h
@@ -51,6 +51,7 @@ class GetEdgesTransformUtils final {
     return scanEdges;
   }
 
+  // [EDGE] AS <e>
   static graph::Project *projectEdges(graph::QueryContext *qctx,
                                       PlanNode *input,
                                       const std::string &colName) {

diff --git a/src/meta/RootUserMan.h b/src/meta/RootUserMan.h
@@ -20,6 +20,26 @@ namespace meta {
  * */
 class RootUserMan {
  public:
+  static ErrorOr<nebula::cpp2::ErrorCode, bool> isGodExists(kvstore::KVStore* kv) {
+    auto rolePrefix = MetaKeyUtils::roleSpacePrefix(kDefaultSpaceId);
+    std::unique_ptr<kvstore::KVIterator> iter;
+    auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, rolePrefix, &iter, false);
+    if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
+      return code;
+    }
+    while (iter->valid()) {
+      auto val = iter->val();
+      auto type = *reinterpret_cast<const meta::cpp2::RoleType*>(val.begin());
+      if (type == meta::cpp2::RoleType::GOD) {
+        LOG(INFO) << "God user exists";
+        return true;
+      }
+      iter->next();
+    }
+    LOG(INFO) << "God user is not exists";
+    return false;
+  }
+
   static bool isUserExists(kvstore::KVStore* kv) {
     auto userKey = MetaKeyUtils::userKey("root");
     std::string val;