diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index 0dc2c6280f3..b136977d6e4 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -45,6 +45,7 @@ nebula_add_library( rule/PushLimitDownProjectRule.cpp rule/EliminateRowCollectRule.cpp rule/PushLimitDownScanAppendVerticesRule.cpp + rule/GetEdgesTransformAppendVerticesLimitRule.cpp rule/GetEdgesTransformRule.cpp rule/PushLimitDownScanEdgesAppendVerticesRule.cpp rule/PushTopNDownIndexScanRule.cpp diff --git a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp new file mode 100644 index 00000000000..22cad3311ec --- /dev/null +++ b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp @@ -0,0 +1,148 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.h" + +#include "common/expression/Expression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/rule/GetEdgesTransformUtils.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/visitor/ExtractFilterExprVisitor.h" + +using nebula::Expression; +using nebula::graph::AppendVertices; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::QueryContext; +using nebula::graph::ScanEdges; +using nebula::graph::ScanVertices; +using nebula::graph::Traverse; + +namespace nebula { +namespace opt { + +std::unique_ptr GetEdgesTransformAppendVerticesLimitRule::kInstance = + std::unique_ptr( + new GetEdgesTransformAppendVerticesLimitRule()); + +GetEdgesTransformAppendVerticesLimitRule::GetEdgesTransformAppendVerticesLimitRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &GetEdgesTransformAppendVerticesLimitRule::pattern() const { + static Pattern pattern = Pattern::create( + PlanNode::Kind::kProject, + {Pattern::create( + PlanNode::Kind::kLimit, + {Pattern::create(PlanNode::Kind::kAppendVertices, + {Pattern::create(PlanNode::Kind::kTraverse, + {Pattern::create(PlanNode::Kind::kScanVertices)})})})}); + return pattern; +} + +bool GetEdgesTransformAppendVerticesLimitRule::match(OptContext *ctx, + const MatchedResult &matched) const { + if (!OptRule::match(ctx, matched)) { + return false; + } + auto traverse = static_cast(matched.planNode({0, 0, 0, 0})); + auto project = static_cast(matched.planNode({0})); + const auto &colNames = traverse->colNames(); + auto colSize = colNames.size(); + DCHECK_GE(colSize, 2); + if (colNames[colSize - 2][0] != '_') { // src + return false; + } + if (traverse->stepRange() != nullptr) { + return false; + } + for (auto yieldColumn : project->columns()->columns()) { + // exclude p=()-[e]->() return p limit 10 + if (yieldColumn->expr()->kind() == nebula::Expression::Kind::kPathBuild) { + return false; + } + } + return true; +} + +StatusOr GetEdgesTransformAppendVerticesLimitRule::transform( + OptContext *ctx, const MatchedResult &matched) const { + auto projectGroupNode = matched.node; + auto project = static_cast(projectGroupNode->node()); + + auto newProject = project->clone(); + auto newProjectGroupNode = OptGroupNode::create(ctx, newProject, projectGroupNode->group()); + newProject->setOutputVar(project->outputVar()); + + auto limitGroupNode = matched.dependencies.front().node; + auto limit = static_cast(limitGroupNode->node()); + + auto newLimit = limit->clone(); + auto newLimitGroup = OptGroup::create(ctx); + auto newLimitGroupNode = newLimitGroup->makeGroupNode(newLimit); + newLimit->setOutputVar(limit->outputVar()); + newProject->setInputVar(newLimit->outputVar()); + + newProjectGroupNode->dependsOn(newLimitGroup); + + auto appendVerticesGroupNode = matched.dependencies.front().dependencies.front().node; + auto appendVertices = static_cast(appendVerticesGroupNode->node()); + auto traverseGroupNode = + matched.dependencies.front().dependencies.front().dependencies.front().node; + auto traverse = static_cast(traverseGroupNode->node()); + auto scanVerticesGroupNode = matched.dependencies.front() + .dependencies.front() + .dependencies.front() + .dependencies.front() + .node; + auto qctx = ctx->qctx(); + + auto newAppendVertices = appendVertices->clone(); + auto newAppendVerticesGroup = OptGroup::create(ctx); + auto colSize = appendVertices->colNames().size(); + newAppendVertices->setOutputVar(appendVertices->outputVar()); + newLimit->setInputVar(newAppendVertices->outputVar()); + newAppendVertices->setColNames( + {appendVertices->colNames()[colSize - 2], appendVertices->colNames()[colSize - 1]}); + auto newAppendVerticesGroupNode = newAppendVerticesGroup->makeGroupNode(newAppendVertices); + + newLimitGroupNode->dependsOn(newAppendVerticesGroup); + + auto *newScanEdges = GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->count(qctx)); + if (newScanEdges == nullptr) { + return TransformResult::noTransform(); + } + auto newScanEdgesGroup = OptGroup::create(ctx); + auto newScanEdgesGroupNode = newScanEdgesGroup->makeGroupNode(newScanEdges); + + auto *newProj = + GetEdgesTransformUtils::projectEdges(qctx, newScanEdges, traverse->colNames().back()); + newProj->setInputVar(newScanEdges->outputVar()); + newProj->setOutputVar(newAppendVertices->inputVar()); + newProj->setColNames({traverse->colNames().back()}); + auto newProjGroup = OptGroup::create(ctx); + auto newProjGroupNode = newProjGroup->makeGroupNode(newProj); + + newAppendVerticesGroupNode->dependsOn(newProjGroup); + newProjGroupNode->dependsOn(newScanEdgesGroup); + for (auto dep : scanVerticesGroupNode->dependencies()) { + newScanEdgesGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseCurr = true; + result.newGroupNodes.emplace_back(newProjectGroupNode); + return result; +} + +std::string GetEdgesTransformAppendVerticesLimitRule::toString() const { + return "GetEdgesTransformAppendVerticesLimitRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.h b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.h new file mode 100644 index 00000000000..75d1bc01346 --- /dev/null +++ b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#pragma once + +#include "graph/optimizer/OptRule.h" + +namespace nebula { + +namespace graph { +class ScanEdges; +class Project; +class Traverse; +class PlanNode; +} // namespace graph + +namespace opt { + +// Convert [[ScanVertices]] to [[ScanEdges]] in certain cases +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Avoid doing Traverse to optimize performance +// Query example: +// 1. match ()-[e]->() return e limit 1 +// +// Tranformation: +// Before: +// +---------+---------+ +// | Project | +// +---------+---------+ +// | +// +---------+---------+ +// | Limit | +// +---------+---------+ +// | +// +---------+---------+ +// | AppendVertices | +// +---------+---------+ +// | +// +---------+---------+ +// | Traverse | +// +---------+---------+ +// | +// +---------+---------+ +// | ScanVertices | +// +---------+---------+ +// +// After: +// +---------+---------+ +// | Project | +// +---------+---------+ +// | +// +---------+---------+ +// | Limit | +// +---------+---------+ +// | +// +---------+---------+ +// | AppendVertices | +// +---------+---------+ +// | +// +---------+---------+ +// | Project | +// +---------+---------+ +// | +// +---------+---------+ +// | ScanEdges | +// +---------+---------+ + +class GetEdgesTransformAppendVerticesLimitRule final : public OptRule { + public: + const Pattern &pattern() const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + StatusOr transform(OptContext *ctx, const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + GetEdgesTransformAppendVerticesLimitRule(); + + static graph::Project *projectEdges(graph::QueryContext *qctx, + graph::PlanNode *input, + const std::string &colName); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp index 885245a338e..aa86c1438e2 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp +++ b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp @@ -8,12 +8,13 @@ #include "common/expression/Expression.h" #include "graph/optimizer/OptContext.h" #include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/rule/GetEdgesTransformUtils.h" #include "graph/planner/plan/PlanNode.h" #include "graph/planner/plan/Query.h" #include "graph/visitor/ExtractFilterExprVisitor.h" using nebula::Expression; -using nebula::graph::AppendVertices; +using nebula::graph::Limit; using nebula::graph::PlanNode; using nebula::graph::Project; using nebula::graph::QueryContext; @@ -32,10 +33,11 @@ GetEdgesTransformRule::GetEdgesTransformRule() { } const Pattern &GetEdgesTransformRule::pattern() const { - static Pattern pattern = - Pattern::create({PlanNode::Kind::kAppendVertices, PlanNode::Kind::kLimit}, - {Pattern::create(PlanNode::Kind::kTraverse, - {Pattern::create(PlanNode::Kind::kScanVertices)})}); + static Pattern pattern = Pattern::create( + PlanNode::Kind::kProject, + {Pattern::create({PlanNode::Kind::kLimit}, + {Pattern::create(PlanNode::Kind::kTraverse, + {Pattern::create(PlanNode::Kind::kScanVertices)})})}); return pattern; } @@ -43,7 +45,8 @@ bool GetEdgesTransformRule::match(OptContext *ctx, const MatchedResult &matched) if (!OptRule::match(ctx, matched)) { return false; } - auto traverse = static_cast(matched.planNode({0, 0})); + auto traverse = static_cast(matched.planNode({0, 0, 0})); + auto project = static_cast(matched.planNode({0})); const auto &colNames = traverse->colNames(); auto colSize = colNames.size(); DCHECK_GE(colSize, 2); @@ -54,49 +57,62 @@ bool GetEdgesTransformRule::match(OptContext *ctx, const MatchedResult &matched) if (traverse->stepRange() != nullptr) { return false; } + for (auto yieldColumn : project->columns()->columns()) { + // exclude p=()-[e]->() return p limit 10 + if (yieldColumn->expr()->kind() == nebula::Expression::Kind::kPathBuild) { + return false; + } + } return true; } StatusOr GetEdgesTransformRule::transform( OptContext *ctx, const MatchedResult &matched) const { - auto appendVerticesOrLimitGroupNode = matched.node; - auto appendVerticesOrLimit = appendVerticesOrLimitGroupNode->node(); - auto traverseGroupNode = matched.dependencies.front().node; + auto projectGroupNode = matched.node; + auto project = static_cast(projectGroupNode->node()); + + auto newProject = project->clone(); + auto newProjectGroupNode = OptGroupNode::create(ctx, newProject, projectGroupNode->group()); + newProject->setOutputVar(project->outputVar()); + + auto limitGroupNode = matched.dependencies.front().node; + auto limit = static_cast(limitGroupNode->node()); + auto traverseGroupNode = matched.dependencies.front().dependencies.front().node; auto traverse = static_cast(traverseGroupNode->node()); - auto scanVerticesGroupNode = matched.dependencies.front().dependencies.front().node; + auto scanVerticesGroupNode = + matched.dependencies.front().dependencies.front().dependencies.front().node; auto qctx = ctx->qctx(); - auto newAppendVerticesOrLimit = appendVerticesOrLimit->clone(); - newAppendVerticesOrLimit->setOutputVar(appendVerticesOrLimit->outputVar()); - if (newAppendVerticesOrLimit->kind() == PlanNode::Kind::kAppendVertices) { - auto colSize = appendVerticesOrLimit->colNames().size(); - newAppendVerticesOrLimit->setColNames({appendVerticesOrLimit->colNames()[colSize - 2], - appendVerticesOrLimit->colNames()[colSize - 1]}); - } - auto newAppendVerticesOrLimitGroupNode = - OptGroupNode::create(ctx, newAppendVerticesOrLimit, appendVerticesOrLimitGroupNode->group()); + auto newLimit = limit->clone(); + auto newLimitGroup = OptGroup::create(ctx); + newLimit->setOutputVar(limit->outputVar()); + + auto newLimitGroupNode = newLimitGroup->makeGroupNode(newLimit); + + newProjectGroupNode->dependsOn(newLimitGroup); - auto *newScanEdges = traverseToScanEdges(traverse); + auto *newScanEdges = GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->count(qctx)); if (newScanEdges == nullptr) { return TransformResult::noTransform(); } auto newScanEdgesGroup = OptGroup::create(ctx); auto newScanEdgesGroupNode = newScanEdgesGroup->makeGroupNode(newScanEdges); - auto *newProj = projectEdges(qctx, newScanEdges, traverse->colNames().back()); + auto *newProj = + GetEdgesTransformUtils::projectEdges(qctx, newScanEdges, traverse->colNames().back()); newProj->setInputVar(newScanEdges->outputVar()); newProj->setOutputVar(traverse->outputVar()); newProj->setColNames({traverse->colNames().back()}); auto newProjGroup = OptGroup::create(ctx); auto newProjGroupNode = newProjGroup->makeGroupNode(newProj); - newAppendVerticesOrLimitGroupNode->dependsOn(newProjGroup); + newLimitGroupNode->dependsOn(newProjGroup); newProjGroupNode->dependsOn(newScanEdgesGroup); newScanEdgesGroupNode->setDeps(scanVerticesGroupNode->dependencies()); TransformResult result; result.eraseAll = true; - result.newGroupNodes.emplace_back(newAppendVerticesOrLimitGroupNode); + result.newGroupNodes.emplace_back(newProjectGroupNode); return result; } @@ -104,47 +120,5 @@ std::string GetEdgesTransformRule::toString() const { return "GetEdgesTransformRule"; } -/*static*/ graph::ScanEdges *GetEdgesTransformRule::traverseToScanEdges( - const graph::Traverse *traverse) { - const auto *edgeProps = traverse->edgeProps(); - if (edgeProps == nullptr) { - return nullptr; - } - for (std::size_t i = 0; i < edgeProps->size(); i++) { - auto type = (*edgeProps)[i].get_type(); - for (std::size_t j = i + 1; j < edgeProps->size(); j++) { - if (type == -((*edgeProps)[j].get_type())) { - // Don't support to retrieve edges of the inbound/outbound together - return nullptr; - } - } - } - auto scanEdges = ScanEdges::make( - traverse->qctx(), - nullptr, - traverse->space(), - edgeProps == nullptr ? nullptr - : std::make_unique>(*edgeProps), - nullptr, - traverse->dedup(), - traverse->limit(), - {}, - traverse->filter() == nullptr ? nullptr : traverse->filter()->clone()); - return scanEdges; -} - -/*static*/ graph::Project *GetEdgesTransformRule::projectEdges(graph::QueryContext *qctx, - PlanNode *input, - const std::string &colName) { - auto *yieldColumns = qctx->objPool()->makeAndAdd(); - auto *edgeExpr = EdgeExpression::make(qctx->objPool()); - auto *listEdgeExpr = ListExpression::make(qctx->objPool()); - listEdgeExpr->setItems({edgeExpr}); - yieldColumns->addColumn(new YieldColumn(listEdgeExpr, colName)); - auto project = Project::make(qctx, input, yieldColumns); - project->setColNames({colName}); - return project; -} - } // namespace opt } // namespace nebula diff --git a/src/graph/optimizer/rule/GetEdgesTransformRule.h b/src/graph/optimizer/rule/GetEdgesTransformRule.h index f570ffbe1d3..6c06907fe13 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformRule.h +++ b/src/graph/optimizer/rule/GetEdgesTransformRule.h @@ -29,9 +29,12 @@ namespace opt { // // Tranformation: // Before: -// // +---------+---------+ -// | AppendVertices | +// | Project | +// +---------+---------+ +// | +// +---------+---------+ +// | Limit | // +---------+---------+ // | // +---------+---------+ @@ -43,9 +46,12 @@ namespace opt { // +---------+---------+ // // After: -// // +---------+---------+ -// | AppendVertices | +// | Project | +// +---------+---------+ +// | +// +---------+---------+ +// | Limit | // +---------+---------+ // | // +---------+---------+ @@ -68,12 +74,6 @@ class GetEdgesTransformRule final : public OptRule { private: GetEdgesTransformRule(); - static graph::ScanEdges *traverseToScanEdges(const graph::Traverse *traverse); - - static graph::Project *projectEdges(graph::QueryContext *qctx, - graph::PlanNode *input, - const std::string &colName); - static std::unique_ptr kInstance; }; diff --git a/src/graph/optimizer/rule/GetEdgesTransformUtils.h b/src/graph/optimizer/rule/GetEdgesTransformUtils.h new file mode 100644 index 00000000000..0a706cddee7 --- /dev/null +++ b/src/graph/optimizer/rule/GetEdgesTransformUtils.h @@ -0,0 +1,70 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_GETEDGESTRANSFORMUTILS_H +#define GRAPH_OPTIMIZER_RULE_GETEDGESTRANSFORMUTILS_H + +#include "graph/optimizer/OptRule.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/visitor/ExtractFilterExprVisitor.h" + +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::ScanEdges; +using nebula::graph::Traverse; + +namespace nebula { + +namespace opt { + +class GetEdgesTransformUtils final { + public: + static graph::ScanEdges *traverseToScanEdges(const graph::Traverse *traverse, + const int64_t limit_count = -1) { + const auto *edgeProps = traverse->edgeProps(); + if (edgeProps == nullptr) { + return nullptr; + } + for (std::size_t i = 0; i < edgeProps->size(); i++) { + auto type = (*edgeProps)[i].get_type(); + for (std::size_t j = i + 1; j < edgeProps->size(); j++) { + if (type == -((*edgeProps)[j].get_type())) { + // Don't support to retrieve edges of the inbound/outbound together + return nullptr; + } + } + } + auto scanEdges = ScanEdges::make( + traverse->qctx(), + nullptr, + traverse->space(), + edgeProps == nullptr ? nullptr + : std::make_unique>(*edgeProps), + nullptr, + traverse->dedup(), + limit_count, + {}, + traverse->filter() == nullptr ? nullptr : traverse->filter()->clone()); + return scanEdges; + } + + static graph::Project *projectEdges(graph::QueryContext *qctx, + PlanNode *input, + const std::string &colName) { + auto *yieldColumns = qctx->objPool()->makeAndAdd(); + auto *edgeExpr = EdgeExpression::make(qctx->objPool()); + auto *listEdgeExpr = ListExpression::make(qctx->objPool()); + listEdgeExpr->setItems({edgeExpr}); + yieldColumns->addColumn(new YieldColumn(listEdgeExpr, colName)); + auto project = Project::make(qctx, input, yieldColumns); + project->setColNames({colName}); + return project; + } +}; + +} // namespace opt +} // namespace nebula +#endif diff --git a/tests/tck/features/bugfix/GetEdgesTransformLimitRule.feature b/tests/tck/features/bugfix/GetEdgesTransformLimitRule.feature new file mode 100644 index 00000000000..17b4ae55fe4 --- /dev/null +++ b/tests/tck/features/bugfix/GetEdgesTransformLimitRule.feature @@ -0,0 +1,23 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Push Limit down scan edges rule + + Background: + Given a graph with space named "student" + + Scenario: Not optimized to ScanEdges + When profiling query: + """ + MATCH p=()-[e]->() + RETURN p LIMIT 3 + """ + Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. + And the execution plan should be: + | id | name | dependencies | operator info | + | 15 | Project | 13 | | + | 13 | Limit | 5 | | + | 5 | AppendVertices | 4 | | + | 4 | Traverse | 12 | | + | 12 | ScanVertices | 3 | {"limit": "-1"} | + | 3 | Start | | |