Merge branch 'master' into support-delete-vertex-without-edge

vesoft-inc · Nov 18, 2021 · 1bbe867 · 1bbe867
2 parents fd8d1bb + 9ebc49d
commit 1bbe867
Show file tree

Hide file tree

Showing 112 changed files with 2,004 additions and 1,799 deletions.
diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production
@@ -102,7 +102,7 @@
 --enable_rocksdb_whole_key_filtering=false
 
 ############### misc ####################
---snapshot_part_rate_limit=8388608
+--snapshot_part_rate_limit=10485760
 --snapshot_batch_size=1048576
 --rebuild_index_part_rate_limit=4194304
 --rebuild_index_batch_size=1048576
diff --git a/src/common/base/SlowOpTracker.cpp b/src/common/base/SlowOpTracker.cpp
@@ -8,4 +8,4 @@
 #include "common/base/Base.h"
 #include "common/time/WallClock.h"
 
-DEFINE_int64(slow_op_threshhold_ms, 50, "default threshhold for slow operation");
+DEFINE_int64(slow_op_threshhold_ms, 100, "default threshhold for slow operation");
diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp
@@ -131,6 +131,16 @@ void Edge::clear() {
   props.clear();
 }
 
+bool Edge::keyEqual(const Edge& rhs) const {
+  if (type != rhs.type && type != -rhs.type) {
+    return false;
+  }
+  if (type == rhs.type) {
+    return src == rhs.src && dst == rhs.dst && ranking == rhs.ranking;
+  }
+  return src == rhs.dst && dst == rhs.src && ranking == rhs.ranking;
+}
+
 }  // namespace nebula
 
 namespace std {

diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h
@@ -68,6 +68,8 @@ struct Edge {
   bool contains(const Value& key) const;
 
   const Value& value(const std::string& key) const;
+
+  bool keyEqual(const Edge& rhs) const;
 };
 
 inline std::ostream& operator<<(std::ostream& os, const Edge& v) { return os << v.toString(); }

diff --git a/src/common/expression/PathBuildExpression.cpp b/src/common/expression/PathBuildExpression.cpp
@@ -26,46 +26,60 @@ const Value& PathBuildExpression::eval(ExpressionContext& ctx) {
 
   for (size_t i = 1; i < items_.size(); ++i) {
     auto& value = items_[i]->eval(ctx);
-    if (value.isEdge()) {
+    if (!buildPath(value, path)) {
+      return Value::kNullBadData;
+    }
+  }
+
+  result_ = path;
+  return result_;
+}
+
+bool PathBuildExpression::buildPath(const Value& value, Path& path) const {
+  switch (value.type()) {
+    case Value::Type::EDGE: {
+      Step step;
       if (!path.steps.empty()) {
-        const auto& lastStep = path.steps.back();
-        const auto& edge = value.getEdge();
-        if (lastStep.dst.vid != edge.src) {
-          return Value::kNullBadData;
-        }
+        getEdge(value, path.steps.back().dst.vid, step);
+      } else {
+        getEdge(value, path.src.vid, step);
       }
-      Step step;
-      getEdge(value, step);
       path.steps.emplace_back(std::move(step));
-    } else if (value.isVertex()) {
+      break;
+    }
+    case Value::Type::VERTEX: {
       if (path.steps.empty()) {
-        return Value::kNullBadData;
+        if (path.src.vid == value.getVertex().vid) {
+          return true;
+        }
+        return false;
       }
       auto& lastStep = path.steps.back();
-      const auto& vert = value.getVertex();
-      if (lastStep.dst.vid != vert.vid) {
-        return Value::kNullBadData;
-      }
       getVertex(value, lastStep.dst);
-    } else if (value.isPath()) {
+      break;
+    }
+    case Value::Type::PATH: {
       const auto& p = value.getPath();
       if (!path.steps.empty()) {
         auto& lastStep = path.steps.back();
-        if (lastStep.dst.vid != p.src.vid) {
-          return Value::kNullBadData;
-        }
         lastStep.dst = p.src;
       }
       path.steps.insert(path.steps.end(), p.steps.begin(), p.steps.end());
-    } else {
-      if ((i & 1) == 1 || path.steps.empty() || !getVertex(value, path.steps.back().dst)) {
-        return Value::kNullBadData;
+      break;
+    }
+    case Value::Type::LIST: {
+      for (const auto& val : value.getList().values) {
+        if (!buildPath(val, path)) {
+          return false;
+        }
       }
+      break;
+    }
+    default: {
+      return false;
     }
   }
-
-  result_ = path;
-  return result_;
+  return true;
 }
 
 bool PathBuildExpression::getVertex(const Value& value, Vertex& vertex) const {
@@ -84,18 +98,26 @@ bool PathBuildExpression::getVertex(const Value& value, Vertex& vertex) const {
   return false;
 }
 
-bool PathBuildExpression::getEdge(const Value& value, Step& step) const {
-  if (value.isEdge()) {
-    const auto& edge = value.getEdge();
+bool PathBuildExpression::getEdge(const Value& value, const Value& lastStepVid, Step& step) const {
+  if (!value.isEdge()) {
+    return false;
+  }
+
+  const auto& edge = value.getEdge();
+  if (lastStepVid == edge.src) {
     step.type = edge.type;
     step.name = edge.name;
     step.ranking = edge.ranking;
     step.props = edge.props;
     step.dst.vid = edge.dst;
-    return true;
+  } else {
+    step.type = -edge.type;
+    step.name = edge.name;
+    step.ranking = edge.ranking;
+    step.props = edge.props;
+    step.dst.vid = edge.src;
   }
-
-  return false;
+  return true;
 }
 
 bool PathBuildExpression::operator==(const Expression& rhs) const {

diff --git a/src/common/expression/PathBuildExpression.h b/src/common/expression/PathBuildExpression.h
@@ -56,7 +56,9 @@ class PathBuildExpression final : public Expression {
 
   bool getVertex(const Value& value, Vertex& vertex) const;
 
-  bool getEdge(const Value& value, Step& step) const;
+  bool getEdge(const Value& value, const Value& lastStepVid, Step& step) const;
+
+  bool buildPath(const Value& value, Path& path) const;
 
  private:
   std::vector<Expression*> items_;

diff --git a/src/common/expression/test/ExpressionContextMock.cpp b/src/common/expression/test/ExpressionContextMock.cpp
@@ -36,9 +36,9 @@ std::unordered_map<std::string, Value> ExpressionContextMock::vals_ = {
     {"srcProperty", Value(13)},
     {"dstProperty", Value(3)},
 
-    {"path_src", Value("1")},
+    {"path_src", Value(Vertex("1", {}))},
     {"path_edge1", Value(Edge("1", "2", 1, "edge", 0, {}))},
-    {"path_v1", Value("2")},
+    {"path_v1", Value(Vertex("2", {}))},
     {"path_edge2", Value(Edge("2", "3", 1, "edge", 0, {}))},
     {"path_v2", Value(Vertex("3", {}))},
     {"path_edge3", Value(Edge("3", "4", 1, "edge", 0, {}))},

diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp
@@ -413,6 +413,7 @@ std::unordered_map<std::string, std::vector<TypeSignature>> FunctionManager::typ
                         Value::Type::FLOAT},
                        Value::Type::LIST),
      }},
+    {"is_edge", {TypeSignature({Value::Type::EDGE}, Value::Type::BOOL)}},
 };
 
 // static
@@ -1945,6 +1946,41 @@ FunctionManager::FunctionManager() {
       }
     };
   }
+  {
+    auto &attr = functions_["none_direct_dst"];
+    attr.minArity_ = 1;
+    attr.maxArity_ = 1;
+    attr.isPure_ = true;
+    attr.body_ = [](const auto &args) -> Value {
+      switch (args[0].get().type()) {
+        case Value::Type::NULLVALUE: {
+          return Value::kNullValue;
+        }
+        case Value::Type::EDGE: {
+          const auto &edge = args[0].get().getEdge();
+          return edge.dst;
+        }
+        case Value::Type::VERTEX: {
+          const auto &v = args[0].get().getVertex();
+          return v.vid;
+        }
+        case Value::Type::LIST: {
+          const auto &listVal = args[0].get().getList();
+          auto &lastVal = listVal.values.back();
+          if (lastVal.isEdge()) {
+            return lastVal.getEdge().dst;
+          } else if (lastVal.isVertex()) {
+            return lastVal.getVertex().vid;
+          } else {
+            return Value::kNullBadType;
+          }
+        }
+        default: {
+          return Value::kNullBadType;
+        }
+      }
+    };
+  }
   {
     auto &attr = functions_["rank"];
     attr.minArity_ = 1;
@@ -2636,6 +2672,13 @@ FunctionManager::FunctionManager() {
       return List(vals);
     };
   }
+  {
+    auto &attr = functions_["is_edge"];
+    attr.minArity_ = 1;
+    attr.maxArity_ = 1;
+    attr.isPure_ = true;
+    attr.body_ = [](const auto &args) -> Value { return args[0].get().isEdge(); };
+  }
 }  // NOLINT
 
 // static

diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h
@@ -457,7 +457,7 @@ class IndexKeyUtils final {
   }
 
   static VertexIDSlice getIndexVertexID(size_t vIdLen, const folly::StringPiece& rawKey) {
-    CHECK_GE(rawKey.size(), kVertexIndexLen + vIdLen);
+    CHECK_GE(rawKey.size(), kTagIndexLen + vIdLen);
     auto offset = rawKey.size() - vIdLen;
     return rawKey.subpiece(offset, vIdLen);
   }

diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp
@@ -34,13 +34,13 @@ std::string NebulaKeyUtils::lastKey(const std::string& prefix, size_t count) {
 }
 
 // static
-std::string NebulaKeyUtils::vertexKey(
+std::string NebulaKeyUtils::tagKey(
     size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId, char pad) {
   CHECK_GE(vIdLen, vId.size());
-  int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVertex);
+  int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kTag_);
 
   std::string key;
-  key.reserve(kVertexLen + vIdLen);
+  key.reserve(kTagLen + vIdLen);
   key.append(reinterpret_cast<const char*>(&item), sizeof(int32_t))
       .append(vId.data(), vId.size())
       .append(vIdLen - vId.size(), pad)
@@ -106,12 +106,12 @@ std::string NebulaKeyUtils::kvKey(PartitionID partId, const folly::StringPiece&
 }
 
 // static
-std::string NebulaKeyUtils::vertexPrefix(size_t vIdLen,
-                                         PartitionID partId,
-                                         const VertexID& vId,
-                                         TagID tagId) {
+std::string NebulaKeyUtils::tagPrefix(size_t vIdLen,
+                                      PartitionID partId,
+                                      const VertexID& vId,
+                                      TagID tagId) {
   CHECK_GE(vIdLen, vId.size());
-  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVertex);
+  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kTag_);
 
   std::string key;
   key.reserve(sizeof(PartitionID) + vIdLen + sizeof(TagID));
@@ -123,9 +123,9 @@ std::string NebulaKeyUtils::vertexPrefix(size_t vIdLen,
 }
 
 // static
-std::string NebulaKeyUtils::vertexPrefix(size_t vIdLen, PartitionID partId, const VertexID& vId) {
+std::string NebulaKeyUtils::tagPrefix(size_t vIdLen, PartitionID partId, const VertexID& vId) {
   CHECK_GE(vIdLen, vId.size());
-  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVertex);
+  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kTag_);
   std::string key;
   key.reserve(sizeof(PartitionID) + vIdLen);
   key.append(reinterpret_cast<const char*>(&item), sizeof(PartitionID))
@@ -135,8 +135,8 @@ std::string NebulaKeyUtils::vertexPrefix(size_t vIdLen, PartitionID partId, cons
 }
 
 // static
-std::string NebulaKeyUtils::vertexPrefix(PartitionID partId) {
-  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVertex);
+std::string NebulaKeyUtils::tagPrefix(PartitionID partId) {
+  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kTag_);
   std::string key;
   key.reserve(sizeof(PartitionID));
   key.append(reinterpret_cast<const char*>(&item), sizeof(PartitionID));
@@ -210,7 +210,7 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
   if (partId == 0) {
     result.emplace_back("");
   } else {
-    result.emplace_back(vertexPrefix(partId));
+    result.emplace_back(tagPrefix(partId));
     result.emplace_back(edgePrefix(partId));
     result.emplace_back(IndexKeyUtils::indexPrefix(partId));
     // kSystem will be written when balance data

diff --git a/src/common/utils/NebulaKeyUtils.h b/src/common/utils/NebulaKeyUtils.h
@@ -11,7 +11,7 @@
 namespace nebula {
 
 /**
- * VertexKeyUtils:
+ * TagKeyUtils:
  * type(1) + partId(3) + vertexId(*) + tagId(4)
  *
  * EdgeKeyUtils:
@@ -55,7 +55,7 @@ class NebulaKeyUtils final {
   /**
    * Generate vertex key for kv store
    * */
-  static std::string vertexKey(
+  static std::string tagKey(
       size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId, char pad = '\0');
 
   static std::string edgeKey(size_t vIdLen,
@@ -75,14 +75,11 @@ class NebulaKeyUtils final {
   /**
    * Prefix for vertex
    * */
-  static std::string vertexPrefix(size_t vIdLen,
-                                  PartitionID partId,
-                                  const VertexID& vId,
-                                  TagID tagId);
+  static std::string tagPrefix(size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId);
 
-  static std::string vertexPrefix(size_t vIdLen, PartitionID partId, const VertexID& vId);
+  static std::string tagPrefix(size_t vIdLen, PartitionID partId, const VertexID& vId);
 
-  static std::string vertexPrefix(PartitionID partId);
+  static std::string tagPrefix(PartitionID partId);
 
   /**
    * Prefix for edge
@@ -111,26 +108,26 @@ class NebulaKeyUtils final {
     return readInt<PartitionID>(rawKey.data(), sizeof(PartitionID)) >> 8;
   }
 
-  static bool isVertex(size_t vIdLen, const folly::StringPiece& rawKey) {
-    if (rawKey.size() != kVertexLen + vIdLen) {
+  static bool isTag(size_t vIdLen, const folly::StringPiece& rawKey) {
+    if (rawKey.size() != kTagLen + vIdLen) {
       return false;
     }
     constexpr int32_t len = static_cast<int32_t>(sizeof(NebulaKeyType));
     auto type = readInt<uint32_t>(rawKey.data(), len) & kTypeMask;
-    return static_cast<NebulaKeyType>(type) == NebulaKeyType::kVertex;
+    return static_cast<NebulaKeyType>(type) == NebulaKeyType::kTag_;
   }
 
   static VertexIDSlice getVertexId(size_t vIdLen, const folly::StringPiece& rawKey) {
-    if (rawKey.size() != kVertexLen + vIdLen) {
-      dumpBadKey(rawKey, kVertexLen + vIdLen, vIdLen);
+    if (rawKey.size() != kTagLen + vIdLen) {
+      dumpBadKey(rawKey, kTagLen + vIdLen, vIdLen);
     }
     auto offset = sizeof(PartitionID);
     return rawKey.subpiece(offset, vIdLen);
   }
 
   static TagID getTagId(size_t vIdLen, const folly::StringPiece& rawKey) {
-    if (rawKey.size() != kVertexLen + vIdLen) {
-      dumpBadKey(rawKey, kVertexLen + vIdLen, vIdLen);
+    if (rawKey.size() != kTagLen + vIdLen) {
+      dumpBadKey(rawKey, kTagLen + vIdLen, vIdLen);
     }
     auto offset = sizeof(PartitionID) + vIdLen;
     return readInt<TagID>(rawKey.data() + offset, sizeof(TagID));