From 37f9956f7526bd17259760775fa1a73e18f446fc Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Tue, 16 Jan 2024 14:32:07 +0000 Subject: [PATCH] refactor(kqp): remove unused pragmas --- ydb/core/kqp/kqp_default_settings.txt | 5 - ydb/core/kqp/opt/logical/kqp_opt_log.cpp | 2 +- ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp | 5 +- ydb/core/kqp/opt/logical/kqp_opt_log_rules.h | 2 +- ydb/core/kqp/provider/yql_kikimr_settings.cpp | 20 - ydb/core/kqp/provider/yql_kikimr_settings.h | 8 - ydb/core/kqp/ut/join/kqp_flip_join_ut.cpp | 630 ++++++------------ ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp | 104 ++- 8 files changed, 266 insertions(+), 510 deletions(-) diff --git a/ydb/core/kqp/kqp_default_settings.txt b/ydb/core/kqp/kqp_default_settings.txt index 48aa4cc4f9d3..3120e3e3c9a4 100644 --- a/ydb/core/kqp/kqp_default_settings.txt +++ b/ydb/core/kqp/kqp_default_settings.txt @@ -63,11 +63,6 @@ DefaultSettings { Value: "1000" } -DefaultSettings { - Name: "OptDisableJoinReverseTableLookup" - Value: "true" -} - DefaultSettings { Name: "_KqpMaxComputeActors" Value: "10000" diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 971f65bc265e..36efc2906c5a 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -146,7 +146,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode JoinToIndexLookup(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpJoinToIndexLookup(node, ctx, KqpCtx, Config); + TExprBase output = KqpJoinToIndexLookup(node, ctx, KqpCtx); DumpAppliedRule("JoinToIndexLookup", node.Ptr(), output.Ptr(), ctx); return output; } diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp index f54c7a3ad848..36af52c789f2 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp @@ -760,8 +760,7 @@ TMaybeNode KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext } // anonymous namespace -TExprBase KqpJoinToIndexLookup(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, - const NYql::TKikimrConfiguration::TPtr& config) +TExprBase KqpJoinToIndexLookup(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { if ((kqpCtx.IsScanQuery() && !kqpCtx.Config->EnableKqpScanQueryStreamIdxLookupJoin) || !node.Maybe()) { return node; @@ -771,7 +770,7 @@ TExprBase KqpJoinToIndexLookup(const TExprBase& node, TExprContext& ctx, const T DBG("-- Join: " << KqpExprToPrettyString(join, ctx)); // SqlIn support (preferred lookup direction) - if (join.JoinType().Value() == "LeftSemi" && !config->HasOptDisableJoinReverseTableLookupLeftSemi()) { + if (join.JoinType().Value() == "LeftSemi") { auto flipJoin = FlipLeftSemiJoin(join, ctx); DBG("-- Flip join"); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h index 003c7fa70bed..97b61b836c0b 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h @@ -25,7 +25,7 @@ NYql::NNodes::TExprBase KqpPushExtractedPredicateToReadTable(NYql::NNodes::TExpr const TKqpOptimizeContext& kqpCtx, NYql::TTypeAnnotationContext& typesCtx); NYql::NNodes::TExprBase KqpJoinToIndexLookup(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config); + const TKqpOptimizeContext& kqpCtx); NYql::NNodes::TExprBase KqpRewriteSqlInToEquiJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp index 7789279ffa8d..c3a4e769f0a0 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp @@ -57,10 +57,6 @@ TKikimrConfiguration::TKikimrConfiguration() { REGISTER_SETTING(*this, EnableLlvm); REGISTER_SETTING(*this, HashJoinMode).Parser([](const TString& v) { return FromString(v); }); - REGISTER_SETTING(*this, OptDisableJoinRewrite); - REGISTER_SETTING(*this, OptDisableJoinTableLookup); - REGISTER_SETTING(*this, OptDisableJoinReverseTableLookup); - REGISTER_SETTING(*this, OptDisableJoinReverseTableLookupLeftSemi); REGISTER_SETTING(*this, OptDisableTopSort); REGISTER_SETTING(*this, OptDisableSqlInToJoin); REGISTER_SETTING(*this, OptEnableInplaceUpdate); @@ -102,22 +98,6 @@ bool TKikimrSettings::DisableLlvmForUdfStages() const { return GetFlagValue(_KqpDisableLlvmForUdfStages.Get()); } -bool TKikimrSettings::HasOptDisableJoinRewrite() const { - return GetFlagValue(OptDisableJoinRewrite.Get()); -} - -bool TKikimrSettings::HasOptDisableJoinTableLookup() const { - return GetFlagValue(OptDisableJoinTableLookup.Get()); -} - -bool TKikimrSettings::HasOptDisableJoinReverseTableLookup() const { - return GetFlagValue(OptDisableJoinReverseTableLookup.Get()); -} - -bool TKikimrSettings::HasOptDisableJoinReverseTableLookupLeftSemi() const { - return GetFlagValue(OptDisableJoinReverseTableLookupLeftSemi.Get()); -} - bool TKikimrSettings::HasOptDisableTopSort() const { return GetFlagValue(OptDisableTopSort.Get()); } diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 9ca375f3d5c1..fd8f09180052 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -51,10 +51,6 @@ struct TKikimrSettings { NCommon::TConfSetting HashJoinMode; /* Disable optimizer rules */ - NCommon::TConfSetting OptDisableJoinRewrite; - NCommon::TConfSetting OptDisableJoinTableLookup; - NCommon::TConfSetting OptDisableJoinReverseTableLookup; - NCommon::TConfSetting OptDisableJoinReverseTableLookupLeftSemi; NCommon::TConfSetting OptDisableTopSort; NCommon::TConfSetting OptDisableSqlInToJoin; NCommon::TConfSetting OptEnableInplaceUpdate; @@ -80,10 +76,6 @@ struct TKikimrSettings { bool SpillingEnabled() const; bool DisableLlvmForUdfStages() const; - bool HasOptDisableJoinRewrite() const; - bool HasOptDisableJoinTableLookup() const; - bool HasOptDisableJoinReverseTableLookup() const; - bool HasOptDisableJoinReverseTableLookupLeftSemi() const; bool HasOptDisableTopSort() const; bool HasOptDisableSqlInToJoin() const; bool HasOptEnableOlapPushdown() const; diff --git a/ydb/core/kqp/ut/join/kqp_flip_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_flip_join_ut.cpp index 6ff608b19bb9..b1a51610441d 100644 --- a/ydb/core/kqp/ut/join/kqp_flip_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_flip_join_ut.cpp @@ -48,23 +48,6 @@ static void CreateSampleTables(TSession session) { static TParams NoParams = TParamsBuilder().Build(); -static const bool EnableJoinFlip = false; -static const bool DisableJoinFlip = true; - -static const char* FormatPragma(bool disableFlip) { - if (!disableFlip) { - return "PRAGMA Kikimr.OptDisableJoinReverseTableLookup = 'False';"; - } - return ""; -} - -static const char* FormatLeftSemiPragma(bool disableFlip) { - if (disableFlip) { - return "PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = 'True';"; - } - return ""; -} - Y_UNIT_TEST_SUITE(KqpFlipJoin) { // simple inner join, only 2 tables @@ -75,30 +58,19 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - // join on key-column of left table and non-key column of right one - const TString query = Sprintf(R"( - %s - SELECT t1.Value, t2.Value - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - ORDER BY t1.Value, t2.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); + // join on key-column of left table and non-key column of right one + const TString query = R"( + SELECT t1.Value, t2.Value + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + ORDER BY t1.Value, t2.Value + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); } // hierarchy of joins, flip on the last layer @@ -109,32 +81,20 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - SELECT t1.Value, t2.Value, t3.Value - FROM `/Root/FJ_Table_3` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk3 - INNER JOIN `/Root/FJ_Table_4` AS t3 ON t2.Key = t3.Key - ORDER BY t1.Value, t2.Value, t3.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, Q_(query), NoParams, - R"([[["Value31"];["Value21"];["Value4_101"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - AssertTableReads(result, "/Root/FJ_Table_4", 1); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - AssertTableReads(result, "/Root/FJ_Table_4", 1); - }); + const TString query = R"( + SELECT t1.Value, t2.Value, t3.Value + FROM `/Root/FJ_Table_3` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk3 + INNER JOIN `/Root/FJ_Table_4` AS t3 ON t2.Key = t3.Key + ORDER BY t1.Value, t2.Value, t3.Value + )"; + + auto result = ExecQueryAndTestResult(session, Q_(query), NoParams, + R"([[["Value31"];["Value21"];["Value4_101"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); + AssertTableReads(result, "/Root/FJ_Table_4", 1); } // hierarchy of joins, flip on the top layer @@ -145,36 +105,24 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - $join = ( - SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key - ); - SELECT t.Value1, t.Value2, t3.Value - FROM `/Root/FJ_Table_3` AS t3 - INNER JOIN $join AS t ON t3.Key = t.Fk - ORDER BY t.Value1, t.Value2, t3.Value - )", FormatPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value11"];["Value21"];["Value31"]];[["Value12"];["Value22"];["Value32"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = Q_(R"( + $join = ( + SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key + ); + SELECT t.Value1, t.Value2, t3.Value + FROM `/Root/FJ_Table_3` AS t3 + INNER JOIN $join AS t ON t3.Key = t.Fk + ORDER BY t.Value1, t.Value2, t3.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[["Value11"];["Value21"];["Value31"]];[["Value12"];["Value22"];["Value32"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // simple left semi join, only 2 tables @@ -185,28 +133,18 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - SELECT t1.Value - FROM `/Root/FJ_Table_1` AS t1 - LEFT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - ORDER BY t1.Value - )", FormatLeftSemiPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value11"]];[["Value12"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 2); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); + const TString query = Q_( + R"( + SELECT t1.Value + FROM `/Root/FJ_Table_1` AS t1 + LEFT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + ORDER BY t1.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value11"]];[["Value12"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 2); + AssertTableReads(result, "/Root/FJ_Table_2", 2); } // hierarchy of joins, flip on the last layer @@ -217,31 +155,20 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - SELECT t1.Key, t1.Value - FROM `/Root/FJ_Table_1` AS t1 - LEFT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - LEFT SEMI JOIN `/Root/FJ_Table_3` AS t3 ON t1.Key = t3.Fk1 - ORDER BY t1.Key, t1.Value - )", FormatLeftSemiPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[[1];["Value11"]];[[2];["Value12"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 2); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = Q_( + R"( + SELECT t1.Key, t1.Value + FROM `/Root/FJ_Table_1` AS t1 + LEFT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + LEFT SEMI JOIN `/Root/FJ_Table_3` AS t3 ON t1.Key = t3.Fk1 + ORDER BY t1.Key, t1.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[[1];["Value11"]];[[2];["Value12"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 2); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // hierarchy of joins, flip on the top layer @@ -252,36 +179,24 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - $join = ( - SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key - ); - SELECT t3.Value + const TString query = Q_( + R"( + $join = ( + SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key + ); + SELECT t3.Value FROM `/Root/FJ_Table_3` AS t3 - LEFT SEMI JOIN $join AS t ON t3.Key = t.Fk - ORDER BY t3.Value - )", FormatLeftSemiPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value31"]];[["Value32"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 2); - }); + LEFT SEMI JOIN $join AS t ON t3.Key = t.Fk + ORDER BY t3.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value31"]];[["Value32"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 2); } // simple right semi join, only 2 tables @@ -292,28 +207,17 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - SELECT t2.Value - FROM `/Root/FJ_Table_1` AS t1 - RIGHT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - ORDER BY t2.Value - )", FormatPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); + const TString query = Q_(R"( + SELECT t2.Value + FROM `/Root/FJ_Table_1` AS t1 + RIGHT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + ORDER BY t2.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); } // hierarchy of joins, flip on the last layer @@ -324,32 +228,20 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - SELECT t3.Key, t3.Value - FROM `/Root/FJ_Table_1` AS t1 - RIGHT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - RIGHT SEMI JOIN `/Root/FJ_Table_3` AS t3 ON t2.Key = t3.Fk2 - ORDER BY t3.Key, t3.Value - )", FormatPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[[1001];["Value31"]];[[1002];["Value32"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = Q_(R"( + SELECT t3.Key, t3.Value + FROM `/Root/FJ_Table_1` AS t1 + RIGHT SEMI JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + RIGHT SEMI JOIN `/Root/FJ_Table_3` AS t3 ON t2.Key = t3.Fk2 + ORDER BY t3.Key, t3.Value + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[[1001];["Value31"]];[[1002];["Value32"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // hierarchy of joins, flip on the top layer @@ -360,36 +252,24 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Q_(Sprintf(R"( - %s - $join = ( - SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key - ); - SELECT t.Value1, t.Value2 - FROM `/Root/FJ_Table_3` AS t3 - RIGHT SEMI JOIN $join AS t ON t3.Key = t.Fk3 - ORDER BY t.Value1, t.Value2 - )", FormatPragma(disableFlip))); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = Q_(R"( + $join = ( + SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key + ); + SELECT t.Value1, t.Value2 + FROM `/Root/FJ_Table_3` AS t3 + RIGHT SEMI JOIN $join AS t ON t3.Key = t.Fk3 + ORDER BY t.Value1, t.Value2 + )"); + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // simple right join, only 2 tables @@ -400,28 +280,17 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - SELECT t2.Value - FROM `/Root/FJ_Table_1` AS t1 - RIGHT JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - ORDER BY t2.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - }); + const TString query = R"( + SELECT t2.Value + FROM `/Root/FJ_Table_1` AS t1 + RIGHT JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + ORDER BY t2.Value + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); } // hierarchy of joins, flip on the last layer @@ -432,32 +301,20 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - SELECT t3.Key, t3.Value - FROM `/Root/FJ_Table_1` AS t1 - RIGHT JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 - RIGHT JOIN `/Root/FJ_Table_3` AS t3 ON t2.Key = t3.Fk2 - ORDER BY t3.Key, t3.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[[1001];["Value31"]];[[1002];["Value32"]];[[1003];["Value33"]];[[1005];["Value35"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = R"( + SELECT t3.Key, t3.Value + FROM `/Root/FJ_Table_1` AS t1 + RIGHT JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk1 + RIGHT JOIN `/Root/FJ_Table_3` AS t3 ON t2.Key = t3.Fk2 + ORDER BY t3.Key, t3.Value + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[[1001];["Value31"]];[[1002];["Value32"]];[[1003];["Value33"]];[[1005];["Value35"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // hierarchy of joins, flip on the top layer @@ -468,36 +325,24 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - $join = ( - SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key - ); - SELECT t.Value1, t.Value2 - FROM `/Root/FJ_Table_3` AS t3 - RIGHT JOIN $join AS t ON t3.Key = t.Fk3 - ORDER BY t.Value1, t.Value2 - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = R"( + $join = ( + SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key + ); + SELECT t.Value1, t.Value2 + FROM `/Root/FJ_Table_3` AS t3 + RIGHT JOIN $join AS t ON t3.Key = t.Fk3 + ORDER BY t.Value1, t.Value2 + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // simple right only join, only 2 tables @@ -508,28 +353,17 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - SELECT t2.Value - FROM `/Root/FJ_Table_3` AS t1 - RIGHT ONLY JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk3 - ORDER BY t2.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = R"( + SELECT t2.Value + FROM `/Root/FJ_Table_3` AS t1 + RIGHT ONLY JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk3 + ORDER BY t2.Value + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])"); + + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // hierarchy of joins, flip on the last layer @@ -540,35 +374,23 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - $join = ( - SELECT t2.Key AS Key, t2.Fk1 AS Fk1, t2.Fk2 AS Fk2, t2.Value AS Value - FROM `/Root/FJ_Table_1` AS t1 - RIGHT ONLY JOIN `/Root/FJ_Table_3` AS t2 ON t1.Key = t2.Fk1 - ); - SELECT t3.Key, t3.Value - FROM $join AS t - INNER JOIN `/Root/FJ_Table_2` AS t3 ON t.Fk2 = t3.Key - ORDER BY t3.Key, t3.Value - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 0); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 0); - AssertTableReads(result, "/Root/FJ_Table_3", 4); - }); + const TString query = R"( + $join = ( + SELECT t2.Key AS Key, t2.Fk1 AS Fk1, t2.Fk2 AS Fk2, t2.Value AS Value + FROM `/Root/FJ_Table_1` AS t1 + RIGHT ONLY JOIN `/Root/FJ_Table_3` AS t2 ON t1.Key = t2.Fk1 + ); + SELECT t3.Key, t3.Value + FROM $join AS t + INNER JOIN `/Root/FJ_Table_2` AS t3 ON t.Fk2 = t3.Key + ORDER BY t3.Key, t3.Value + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 0); + AssertTableReads(result, "/Root/FJ_Table_3", 4); } // hierarchy of joins, flip on the top layer @@ -579,36 +401,24 @@ Y_UNIT_TEST_SUITE(KqpFlipJoin) { CreateSampleTables(session); - auto test = [&](bool disableFlip, std::function assertFn) { - const TString query = Sprintf(R"( - %s - $join = ( - SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 - FROM `/Root/FJ_Table_1` AS t1 - INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key - ); - SELECT t.Value1, t.Value2 - FROM `/Root/FJ_Table_4` AS t3 - RIGHT ONLY JOIN $join AS t ON t3.Key = t.Fk3 - ORDER BY t.Value1, t.Value2 - )", FormatPragma(disableFlip)); - - auto result = ExecQueryAndTestResult(session, query, NoParams, - R"([[["Value12"];["Value22"]]])"); - assertFn(result); - }; - - test(DisableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_4", 3); - }); - - test(EnableJoinFlip, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/FJ_Table_1", 4); - AssertTableReads(result, "/Root/FJ_Table_2", 2); - AssertTableReads(result, "/Root/FJ_Table_4", 3); - }); + const TString query = R"( + $join = ( + SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3 + FROM `/Root/FJ_Table_1` AS t1 + INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Fk2 = t2.Key + ); + SELECT t.Value1, t.Value2 + FROM `/Root/FJ_Table_4` AS t3 + RIGHT ONLY JOIN $join AS t ON t3.Key = t.Fk3 + ORDER BY t.Value1, t.Value2 + )"; + + auto result = ExecQueryAndTestResult(session, query, NoParams, + R"([[["Value12"];["Value22"]]])"); + + AssertTableReads(result, "/Root/FJ_Table_1", 4); + AssertTableReads(result, "/Root/FJ_Table_2", 2); + AssertTableReads(result, "/Root/FJ_Table_4", 3); } } // Y_UNIT_TEST_SUITE diff --git a/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp b/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp index 9b6b4025b408..23fa8ad05e3b 100644 --- a/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_sqlin_ut.cpp @@ -508,40 +508,31 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); + auto query = Q1_( + R"( + PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; + DECLARE $in AS List>; + SELECT Group AS g, Amount AS a, Comment + FROM `/Root/Test` + WHERE (Group, Name) IN (SELECT (k, v) FROM AS_TABLE($in)) -- table source + ORDER BY g, a + )"); - auto test = [&](bool disableOpt, std::function assertFn) { - auto query = TStringBuilder() - << Q1_(Sprintf(R"( - PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; - PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s"; -- not depends on `Kikimr.OptDisableSqlInToJoin` pragma - DECLARE $in AS List>; - SELECT Group AS g, Amount AS a, Comment - FROM `/Root/Test` - WHERE (Group, Name) IN (SELECT (k, v) FROM AS_TABLE($in)) -- table source - ORDER BY g, a - )", disableOpt ? "True" : "False")); - - auto params = TParamsBuilder().AddParam("$in").BeginList() - .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Anna").EndStruct() - .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Jack").EndStruct() - .AddListItem().BeginStruct().AddMember("k").Uint32(4).AddMember("v").String("Hugo").EndStruct() - .AddListItem().BeginStruct().AddMember("k").Uint32(9).AddMember("v").String("Harry").EndStruct() - .EndList().Build().Build(); - - auto result = ExecQueryAndTestResult(session, query, params, - R"([[[1u];[3500u];["None"]]; - [[1u];[100500u];["Just Jack"]]; - [[4u];[77u];["Boss"]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/Test", 8); - }); + auto params = TParamsBuilder().AddParam("$in").BeginList() + .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Anna").EndStruct() + .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Jack").EndStruct() + .AddListItem().BeginStruct().AddMember("k").Uint32(4).AddMember("v").String("Hugo").EndStruct() + .AddListItem().BeginStruct().AddMember("k").Uint32(9).AddMember("v").String("Harry").EndStruct() + .EndList().Build().Build(); - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/Test", 3); - }); + auto result = ExecQueryAndTestResult(session, query, params, + R"( + [[[1u];[3500u];["None"]]; + [[1u];[100500u];["Just Jack"]]; + [[4u];[77u];["Boss"]]] + )"); + + AssertTableReads(result, "/Root/Test", 8); } Y_UNIT_TEST(SelectNotAllElements) { @@ -861,38 +852,27 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); + const TString query = Q1_( + R"( + PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; + DECLARE $in AS List>; + SELECT Value + FROM `/Root/SecondaryComplexKeys` VIEW Index + WHERE (Fk1, Fk2) IN (SELECT (k, v) FROM AS_TABLE($in)) + ORDER BY Value + )"); - auto test = [&](bool disableOpt, std::function assertFn) { - const TString query = Q1_(Sprintf(R"( - PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; - PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s"; - DECLARE $in AS List>; - SELECT Value - FROM `/Root/SecondaryComplexKeys` VIEW Index - WHERE (Fk1, Fk2) IN (SELECT (k, v) FROM AS_TABLE($in)) - ORDER BY Value - )", disableOpt ? "True" : "False")); - - auto params = TParamsBuilder().AddParam("$in").BeginList() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).AddMember("v").String("Fk2").EndStruct() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).AddMember("v").String("Fk5").EndStruct() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).AddMember("v").String("FkNull").EndStruct() - .EndList().Build().Build(); - - auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); - }); + auto params = TParamsBuilder().AddParam("$in").BeginList() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).AddMember("v").String("Fk2").EndStruct() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).AddMember("v").String("Fk5").EndStruct() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).AddMember("v").String("FkNull").EndStruct() + .EndList().Build().Build(); - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); - }); + auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])"); + + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); } Y_UNIT_TEST(TupleNotOnlyOfKeys) {