-
Notifications
You must be signed in to change notification settings - Fork 606
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
30 changed files
with
920 additions
and
136 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| #include "kqp_opt_cbo.h" | ||
| #include "kqp_opt_log_impl.h" | ||
|
|
||
| #include <ydb/library/yql/core/yql_opt_utils.h> | ||
| #include <ydb/library/yql/utils/log/log.h> | ||
|
|
||
|
|
||
| namespace NKikimr::NKqp::NOpt { | ||
|
|
||
| using namespace NYql; | ||
| using namespace NYql::NCommon; | ||
| using namespace NYql::NDq; | ||
| using namespace NYql::NNodes; | ||
|
|
||
| namespace { | ||
|
|
||
| /** | ||
| * KQP specific rule to check if a LookupJoin is applicable | ||
| */ | ||
| bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNode>& node, const TVector<TString>& joinColumns, const TKqpProviderContext& ctx) { | ||
|
|
||
| auto rel = std::static_pointer_cast<TKqpRelOptimizerNode>(node); | ||
| auto expr = TExprBase(rel->Node); | ||
|
|
||
| if (ctx.KqpCtx.IsScanQuery() && !ctx.KqpCtx.Config->EnableKqpScanQueryStreamIdxLookupJoin) { | ||
| return false; | ||
| } | ||
|
|
||
| if (find_if(joinColumns.begin(), joinColumns.end(), [&] (const TString& s) { return node->Stats->KeyColumns[0] == s;})) { | ||
| return true; | ||
| } | ||
|
|
||
| auto readMatch = MatchRead<TKqlReadTable>(expr); | ||
| TMaybeNode<TKqlKeyInc> maybeTablePrefix; | ||
| size_t prefixSize; | ||
|
|
||
| if (readMatch) { | ||
| if (readMatch->FlatMap && !IsPassthroughFlatMap(readMatch->FlatMap.Cast(), nullptr)){ | ||
| return false; | ||
| } | ||
| auto read = readMatch->Read.Cast<TKqlReadTable>(); | ||
| maybeTablePrefix = GetRightTableKeyPrefix(read.Range()); | ||
|
|
||
| if (!maybeTablePrefix) { | ||
| return false; | ||
| } | ||
|
|
||
| prefixSize = maybeTablePrefix.Cast().ArgCount(); | ||
|
|
||
| if (!prefixSize) { | ||
| return true; | ||
| } | ||
| } | ||
| else { | ||
| readMatch = MatchRead<TKqlReadTableRangesBase>(expr); | ||
| if (readMatch) { | ||
| if (readMatch->FlatMap && !IsPassthroughFlatMap(readMatch->FlatMap.Cast(), nullptr)){ | ||
| return false; | ||
| } | ||
| auto read = readMatch->Read.Cast<TKqlReadTableRangesBase>(); | ||
| if (TCoVoid::Match(read.Ranges().Raw())) { | ||
| return true; | ||
| } else { | ||
| auto prompt = TKqpReadTableExplainPrompt::Parse(read); | ||
|
|
||
| if (prompt.PointPrefixLen != prompt.UsedKeyColumns.size()) { | ||
| return false; | ||
| } | ||
|
|
||
| if (prompt.ExpectedMaxRanges != TMaybe<ui64>(1)) { | ||
| return false; | ||
| } | ||
| prefixSize = prompt.PointPrefixLen; | ||
| } | ||
| } | ||
| } | ||
| if (! readMatch) { | ||
| return false; | ||
| } | ||
|
|
||
| if (prefixSize < node->Stats->KeyColumns.size() && !(find_if(joinColumns.begin(), joinColumns.end(), [&] (const TString& s) { | ||
| return node->Stats->KeyColumns[prefixSize] == s; | ||
| }))){ | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| bool IsLookupJoinApplicable(std::shared_ptr<IBaseOptimizerNode> left, | ||
| std::shared_ptr<IBaseOptimizerNode> right, | ||
| const std::set<std::pair<TJoinColumn, TJoinColumn>>& joinConditions, | ||
| TKqpProviderContext& ctx) { | ||
|
|
||
| Y_UNUSED(left); | ||
|
|
||
| auto rightStats = right->Stats; | ||
|
|
||
| if (rightStats->Type != EStatisticsType::BaseTable) { | ||
| return false; | ||
| } | ||
| if (joinConditions.size() > rightStats->KeyColumns.size()) { | ||
| return false; | ||
| } | ||
|
|
||
| for (auto [leftCol, rightCol] : joinConditions) { | ||
| if (! find_if(rightStats->KeyColumns.begin(), rightStats->KeyColumns.end(), | ||
| [rightCol] (const TString& s) { | ||
| return rightCol.AttributeName == s; | ||
| } )) { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| TVector<TString> joinKeys; | ||
| for( auto [leftJc, rightJc] : joinConditions ) { | ||
| joinKeys.emplace_back( rightJc.AttributeName); | ||
| } | ||
|
|
||
| return IsLookupJoinApplicableDetailed(std::static_pointer_cast<TRelOptimizerNode>(right), joinKeys, ctx); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr<IBaseOptimizerNode>& left, | ||
| const std::shared_ptr<IBaseOptimizerNode>& right, | ||
| const std::set<std::pair<NDq::TJoinColumn, NDq::TJoinColumn>>& joinConditions, | ||
| EJoinAlgoType joinAlgo) { | ||
|
|
||
| switch( joinAlgo ) { | ||
| case EJoinAlgoType::LookupJoin: | ||
| if (OptLevel==2 && left->Stats->Nrows > 10e3) { | ||
| return false; | ||
| } | ||
| return IsLookupJoinApplicable(left, right, joinConditions, *this); | ||
|
|
||
| case EJoinAlgoType::DictJoin: | ||
| return right->Stats->Nrows < 10e5; | ||
| case EJoinAlgoType::MapJoin: | ||
| return right->Stats->Nrows < 10e6; | ||
| case EJoinAlgoType::GraceJoin: | ||
| return true; | ||
| } | ||
| } | ||
|
|
||
| double TKqpProviderContext::ComputeJoinCost(const TOptimizerStatistics& leftStats, const TOptimizerStatistics& rightStats, EJoinAlgoType joinAlgo) const { | ||
|
|
||
| switch(joinAlgo) { | ||
| case EJoinAlgoType::LookupJoin: | ||
| if (OptLevel==1) { | ||
| return -1; | ||
| } | ||
| return leftStats.Nrows; | ||
| case EJoinAlgoType::DictJoin: | ||
| return leftStats.Nrows + 1.7 * rightStats.Nrows; | ||
| case EJoinAlgoType::MapJoin: | ||
| return leftStats.Nrows + 1.8 * rightStats.Nrows; | ||
| case EJoinAlgoType::GraceJoin: | ||
| return leftStats.Nrows + 2.0 * rightStats.Nrows; | ||
| } | ||
| } | ||
|
|
||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| #pragma once | ||
|
|
||
| #include <ydb/library/yql/ast/yql_expr.h> | ||
| #include <ydb/library/yql/core/cbo/cbo_optimizer_new.h> | ||
|
|
||
| #include <ydb/core/kqp/opt/kqp_opt.h> | ||
|
|
||
| namespace NKikimr::NKqp::NOpt { | ||
|
|
||
| /** | ||
| * KQP specific Rel node, includes a pointer to ExprNode | ||
| */ | ||
| struct TKqpRelOptimizerNode : public NYql::TRelOptimizerNode { | ||
| const NYql::TExprNode::TPtr Node; | ||
|
|
||
| TKqpRelOptimizerNode(TString label, std::shared_ptr<NYql::TOptimizerStatistics> stats, const NYql::TExprNode::TPtr node) : | ||
| TRelOptimizerNode(label, stats), Node(node) { } | ||
| }; | ||
|
|
||
| /** | ||
| * KQP Specific cost function and join applicability cost function | ||
| */ | ||
| struct TKqpProviderContext : public NYql::IProviderContext { | ||
| TKqpProviderContext(const TKqpOptimizeContext& kqpCtx, const int optLevel) : KqpCtx(kqpCtx), OptLevel(optLevel) {} | ||
|
|
||
| virtual bool IsJoinApplicable(const std::shared_ptr<NYql::IBaseOptimizerNode>& left, | ||
| const std::shared_ptr<NYql::IBaseOptimizerNode>& right, | ||
| const std::set<std::pair<NYql::NDq::TJoinColumn, NYql::NDq::TJoinColumn>>& joinConditions, | ||
| NYql::EJoinAlgoType joinAlgo) override; | ||
|
|
||
| virtual double ComputeJoinCost(const NYql::TOptimizerStatistics& leftStats, const NYql::TOptimizerStatistics& rightStats, NYql::EJoinAlgoType joinAlgo) const override; | ||
|
|
||
| const TKqpOptimizeContext& KqpCtx; | ||
| int OptLevel; | ||
| }; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,14 @@ | ||
| #pragma once | ||
|
|
||
| #include <ydb/core/kqp/opt/kqp_opt.h> | ||
| #include <ydb/core/kqp/opt/logical/kqp_opt_cbo.h> | ||
|
|
||
| namespace NKikimr::NKqp::NOpt { | ||
|
|
||
| struct TKqpOptimizeContext; | ||
|
|
||
| TAutoPtr<NYql::IGraphTransformer> CreateKqpLogOptTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx, | ||
| NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config); | ||
| NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config, | ||
| NKikimr::NKqp::NOpt::TKqpProviderContext& pctx); | ||
|
|
||
| } // namespace NKikimr::NKqp::NOpt |
Oops, something went wrong.