Merge branch 'master' into fix-client-connection

.github/workflows/nightly.yml

@@ -33,6 +33,9 @@ jobs:
         with:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.7'
       - name: package
         run: ./package/package.sh
       - name: output some vars

.github/workflows/rc.yml

@@ -37,6 +37,9 @@ jobs:
         with:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.7'
       - uses: ./.github/actions/tagname-action
         id: tag
       - name: package

CMakeLists.txt

@@ -25,6 +25,7 @@ project("Nebula Graph" C CXX)
 
 option(ENABLE_PACK_ONE "Whether to package into one" ON)
 option(ENABLE_VERBOSE_BISON "Enable Bison to report state" OFF)
+option(ENABLE_CONSOLE_COMPILATION "Enable nebula-console compilation" OFF)
 option(ENABLE_PACKAGE_TAR "Enable package artifacts to tar." OFF)
 option(ENABLE_CREATE_GIT_HOOKS "Enable create git hooks." ON)
 option(ENABLE_INCLUDE_WHAT_YOU_USE "Enable include-what-you-use find  nouse include files" OFF)

cmake/CPackage.cmake

@@ -162,9 +162,11 @@ macro(package to_one name home_page scripts_dir)
     cpack_add_component(storage GROUP storage DEPENDS common)
     cpack_add_component(meta GROUP meta DEPENDS common)
     cpack_add_component(tool GROUP tool)
+    cpack_add_component(nebula-console GROUP nebula-console)
     cpack_add_component_group(common)
     cpack_add_component_group(graph)
     cpack_add_component_group(storage)
     cpack_add_component_group(meta)
     cpack_add_component_group(tool)
+    cpack_add_component_group(nebula-console)
 endmacro()

package/package.sh

@@ -112,6 +112,7 @@ function _build_graph {
           -DENABLE_STATIC_UBSAN=${ssan} \
           -DCMAKE_INSTALL_PREFIX=${install_prefix} \
           -DENABLE_TESTING=OFF \
+          -DENABLE_CONSOLE_COMPILATION=ON \
           -DENABLE_PACK_ONE=${package_one} \
           -DENABLE_COMPRESSED_DEBUG_INFO=${enable_compressed_debug_info} \
           -DENABLE_PACKAGE_TAR=${package_tar} \

src/CMakeLists.txt

@@ -16,3 +16,4 @@ nebula_add_subdirectory(storage)
 nebula_add_subdirectory(tools)
 nebula_add_subdirectory(version)
 nebula_add_subdirectory(webservice)
+nebula_add_subdirectory(console)

src/clients/meta/MetaClient.cpp

@@ -114,12 +114,6 @@ bool MetaClient::isMetadReady() {
 }
 
 bool MetaClient::waitForMetadReady(int count, int retryIntervalSecs) {
-  auto status = verifyVersion();
-  if (!status.ok()) {
-    LOG(ERROR) << status;
-    return false;
-  }
-
   if (!options_.skipConfig_) {
     std::string gflagsJsonPath;
     GflagsManager::getGflagsModule(gflagsModule_);
@@ -137,6 +131,11 @@ bool MetaClient::waitForMetadReady(int count, int retryIntervalSecs) {
     LOG(ERROR) << "Connect to the MetaServer Failed";
     return false;
   }
+  auto status = verifyVersion();
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return false;
+  }
 
   CHECK(bgThread_->start());
   LOG(INFO) << "Register time task for heartbeat!";

src/console/CMakeLists.txt

@@ -0,0 +1,35 @@
+if(ENABLE_CONSOLE_COMPILATION)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} symbolic-ref --short HEAD 
+    OUTPUT_VARIABLE git_branch
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  )
+
+  add_custom_command(
+    OUTPUT ${CMAKE_BINARY_DIR}/nebula-console
+    COMMAND bash ${CMAKE_SOURCE_DIR}/src/console/compile-console.sh ${git_branch} ${CMAKE_BINARY_DIR}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  )
+
+  add_custom_target(GenerateConsole ALL
+    DEPENDS ${CMAKE_BINARY_DIR}/nebula-console
+  )
+
+  add_dependencies(nebula-graphd GenerateConsole)
+
+  install(
+    FILES
+        ${CMAKE_BINARY_DIR}/nebula-console
+    PERMISSIONS
+        OWNER_EXECUTE OWNER_WRITE OWNER_READ
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ
+    DESTINATION
+        bin
+    COMPONENT
+        nebula-console
+    OPTIONAL
+)
+endif()

src/console/compile-console.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Copyright (c) 2020 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+
+git clone -b $1 https://github.com/vesoft-inc/nebula-console.git && \
+pushd nebula-console && \
+make && cp nebula-console $2/ && \
+popd && \
+rm -rf nebula-console

src/graph/planner/plan/PlanNodeVisitor.h

@@ -22,7 +22,6 @@ class PlanNodeVisitor {
   virtual void visit(Aggregate *node) = 0;
   virtual void visit(Traverse *node) = 0;
   virtual void visit(AppendVertices *node) = 0;
-  virtual void visit(Dedup *node) = 0;
   virtual void visit(BiJoin *node) = 0;
 };
 

src/graph/planner/plan/Query.cpp

@@ -544,10 +544,6 @@ Dedup::Dedup(QueryContext* qctx, PlanNode* input) : SingleInputNode(qctx, Kind::
   copyInputColNames(input);
 }
 
-void Dedup::accept(PlanNodeVisitor* visitor) {
-  visitor->visit(this);
-}
-
 PlanNode* Dedup::clone() const {
   auto* newDedup = Dedup::make(qctx_, nullptr);
   newDedup->cloneMembers(*this);

src/graph/planner/plan/Query.h

@@ -1168,8 +1168,6 @@ class Dedup final : public SingleInputNode {
     return qctx->objPool()->add(new Dedup(qctx, input));
   }
 
-  void accept(PlanNodeVisitor* visitor) override;
-
   PlanNode* clone() const override;
 
  private:

src/graph/visitor/ExtractFilterExprVisitor.cpp

@@ -61,17 +61,9 @@ void ExtractFilterExprVisitor::visit(InputPropertyExpression *) {
 }
 
 void ExtractFilterExprVisitor::visit(LabelTagPropertyExpression *) {
-  switch (pushType_) {
-    case PushType::kGetNeighbors:
-      canBePushed_ = false;
-      break;
-    case PushType::kGetVertices:
-      canBePushed_ = true;
-      break;
-    case PushType::kGetEdges:
-      canBePushed_ = false;
-      break;
-  }
+  // Storage don't support this expression
+  // Convert it to tag/vertex related property expression for before push-down
+  canBePushed_ = false;
 }
 
 void ExtractFilterExprVisitor::visit(VariablePropertyExpression *) {

src/graph/visitor/PrunePropertiesVisitor.cpp

@@ -252,14 +252,6 @@ void PrunePropertiesVisitor::visit(AppendVertices *node) {
   status_ = depsPruneProperties(node->dependencies());
 }
 
-void PrunePropertiesVisitor::visit(Dedup *node) {
-  const auto &colNames = qctx_->symTable()->getVar(node->inputVar())->colNames;
-  for (auto &colName : colNames) {
-    propsUsed_.colsSet.insert(colName);
-  }
-  status_ = depsPruneProperties(node->dependencies());
-}
-
 void PrunePropertiesVisitor::visit(BiJoin *node) {
   for (auto *hashKey : node->hashKeys()) {
     status_ = extractPropsFromExpr(hashKey);

src/graph/visitor/PrunePropertiesVisitor.h

@@ -34,7 +34,6 @@ class PrunePropertiesVisitor final : public PlanNodeVisitor {
   void visit(Aggregate *node) override;
   void visit(Traverse *node) override;
   void visit(AppendVertices *node) override;
-  void visit(Dedup *node) override;
   void visit(BiJoin *node) override;
 
  private:

src/graph/visitor/test/ExtractFilterExprVisitorTest.cpp

@@ -464,5 +464,12 @@ TEST_F(ExtractFilterExprVisitorTest, TestMultiCanNotPush) {
   }
 }
 
+TEST_F(ExtractFilterExprVisitorTest, TestPushLabelTagPropExpr) {
+  // $-.input.tag.prop can't push down
+  auto expr = LabelTagPropertyExpression::make(
+      pool_.get(), InputPropertyExpression::make(pool_.get(), "input"), "tag", "prop");
+  checkIfEqual(expr, expr->clone(), nullptr, false);
+}
+
 }  // namespace graph
 }  // namespace nebula

src/storage/admin/StatsTask.cpp

@@ -101,14 +101,16 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
   auto partitionNum = partitionNumRet.value();
   LOG(INFO) << "Start stats task";
   CHECK_NOTNULL(env_->kvstore_);
-  auto vertexPrefix = NebulaKeyUtils::tagPrefix(part);
-  std::unique_ptr<kvstore::KVIterator> vertexIter;
+  auto tagPrefix = NebulaKeyUtils::tagPrefix(part);
+  std::unique_ptr<kvstore::KVIterator> tagIter;
   auto edgePrefix = NebulaKeyUtils::edgePrefix(part);
   std::unique_ptr<kvstore::KVIterator> edgeIter;
+  auto vertexPrefix = NebulaKeyUtils::vertexPrefix(part);
+  std::unique_ptr<kvstore::KVIterator> vertexIter;
 
   // When the storage occurs leader change, continue to read data from the
   // follower instead of reporting an error.
-  auto ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true);
+  auto ret = env_->kvstore_->prefix(spaceId, part, tagPrefix, &tagIter, true);
   if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
     LOG(INFO) << "Stats task failed";
     return ret;
@@ -118,7 +120,11 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     LOG(INFO) << "Stats task failed";
     return ret;
   }
-
+  ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true);
+  if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+    LOG(INFO) << "Stats task failed";
+    return ret;
+  }
   std::unordered_map<TagID, int64_t> tagsVertices;
   std::unordered_map<EdgeType, int64_t> edgetypeEdges;
   std::unordered_map<PartitionID, int64_t> positiveRelevancy;
@@ -134,8 +140,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     edgetypeEdges[edge.first] = 0;
   }
 
-  VertexID lastVertexId = "";
-
   // Only stats valid vertex data, no multi version
   // For example
   // Vid  tagId
@@ -144,31 +148,24 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
   // 2     3
   // 2     5
   // 3     1
-  while (vertexIter && vertexIter->valid()) {
+  while (tagIter && tagIter->valid()) {
     if (UNLIKELY(canceled_)) {
       LOG(INFO) << "Stats task is canceled";
       return nebula::cpp2::ErrorCode::E_USER_CANCEL;
     }
 
-    auto key = vertexIter->key();
+    auto key = tagIter->key();
     auto vId = NebulaKeyUtils::getVertexId(vIdLen, key).str();
     auto tagId = NebulaKeyUtils::getTagId(vIdLen, key);
 
     auto it = tagsVertices.find(tagId);
     if (it == tagsVertices.end()) {
       // Invalid data
-      vertexIter->next();
+      tagIter->next();
       continue;
     }
-
-    if (vId == lastVertexId) {
-      tagsVertices[tagId] += 1;
-    } else {
-      tagsVertices[tagId] += 1;
-      spaceVertices++;
-      lastVertexId = vId;
-    }
-    vertexIter->next();
+    tagsVertices[tagId] += 1;
+    tagIter->next();
   }
 
   // Only stats valid edge data, no multi version
@@ -219,7 +216,10 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     }
     edgeIter->next();
   }
-
+  while (vertexIter && vertexIter->valid()) {
+    spaceVertices++;
+    vertexIter->next();
+  }
   nebula::meta::cpp2::StatsItem statsItem;
 
   // convert tagId/edgeType to tagName/edgeName

tests/tck/features/optimizer/PrunePropertiesRule.feature

@@ -124,6 +124,26 @@ Feature: Prune Properties rule
       | ("Kyle Anderson" :player{age: 25, name: "Kyle Anderson"})   | 25  |
       | ("Damian Lillard" :player{age: 28, name: "Damian Lillard"}) | 28  |
       | ("Damian Lillard" :player{age: 28, name: "Damian Lillard"}) | 28  |
+    When executing query:
+      """
+      MATCH (v:player{name: "Tony Parker"})-[:serve]->(t:team)
+      WITH DISTINCT v, t
+      RETURN t
+      """
+    Then the result should be, in any order:
+      | t                                  |
+      | ("Spurs" :team{name: "Spurs"})     |
+      | ("Hornets" :team{name: "Hornets"}) |
+    When executing query:
+      """
+      MATCH (v:player{name: "Tony Parker"})-[:serve]->(t:team)
+      WITH DISTINCT v.player.age as age, t
+      RETURN t
+      """
+    Then the result should be, in any order:
+      | t                                  |
+      | ("Spurs" :team{name: "Spurs"})     |
+      | ("Hornets" :team{name: "Hornets"}) |
 
   Scenario: Multi Path Patterns
     When profiling query: