From 0ee8cb79e5871b590cad9c8ddc6d6e69fee989d8 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 6 Feb 2025 16:05:49 +0300 Subject: [PATCH] YDB FQ: introduce new `TGenTable` expr node (#14228) --- .../expr_nodes/yql_generic_expr_nodes.json | 11 ++++++- .../provider/yql_generic_datasource.cpp | 2 +- .../yql_generic_datasource_type_ann.cpp | 30 +++++++++++++++++-- .../provider/yql_generic_dq_integration.cpp | 2 +- .../provider/yql_generic_load_meta.cpp | 24 +++++++++------ .../provider/yql_generic_physical_opt.cpp | 2 +- 6 files changed, 56 insertions(+), 15 deletions(-) diff --git a/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json b/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json index 90652bc3037e..aa52c14f5576 100644 --- a/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json +++ b/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json @@ -24,6 +24,15 @@ {"Index": 1, "Name": "DataSource", "Type": "TGenDataSource"} ] }, + { + "Name": "TGenTable", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "GenTable"}, + "Children": [ + {"Index": 0, "Name": "Name", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Splits", "Type": "TExprBase"} + ] + }, { "Name": "TGenReadTable", "Base": "TCallable", @@ -31,7 +40,7 @@ "Children": [ {"Index": 0, "Name": "World", "Type": "TExprBase"}, {"Index": 1, "Name": "DataSource", "Type": "TGenDataSource"}, - {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Table", "Type": "TGenTable"}, {"Index": 3, "Name": "Columns", "Type": "TExprBase"}, {"Index": 4, "Name": "FilterPredicate", "Type": "TCoLambda"} ] diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_datasource.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_datasource.cpp index 70ddd04ed5b5..a272e07aba76 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_datasource.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_datasource.cpp @@ -126,7 +126,7 @@ namespace NYql { auto cluster = dataSource.Cast().Cluster(); tableNameBuilder << cluster.Value() << "."; } - tableNameBuilder << '`' << maybeTable.Cast().Value() << '`'; + tableNameBuilder << '`' << maybeTable.Cast().Name().Value() << '`'; inputs.push_back( TPinInfo(maybeRead.DataSource().Raw(), nullptr, maybeTable.Cast().Raw(), tableNameBuilder, false)); return 1; diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp index 4d92e8ff4006..eb273195a256 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp @@ -27,6 +27,7 @@ namespace NYql { { using TSelf = TGenericDataSourceTypeAnnotationTransformer; AddHandler({TCoConfigure::CallableName()}, Hndl(&TSelf::HandleConfig)); + AddHandler({TGenTable::CallableName()}, Hndl(&TSelf::HandleTable)); AddHandler({TGenReadTable::CallableName()}, Hndl(&TSelf::HandleReadTable)); AddHandler({TGenSourceSettings::CallableName()}, Hndl(&TSelf::HandleSourceSettings)); } @@ -48,6 +49,19 @@ namespace NYql { return TStatus::Ok; } + TStatus HandleTable(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(*input->Child(TGenTable::idx_Name), ctx)) { + return TStatus::Error; + } + + input->SetTypeAnn(ctx.MakeType()); + return TStatus::Ok; + } + TStatus HandleSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { if (!EnsureArgsCount(*input, 6, ctx)) { return TStatus::Error; @@ -131,7 +145,7 @@ namespace NYql { return TStatus::Error; } - if (!EnsureAtom(*input->Child(TGenReadTable::idx_Table), ctx)) { + if (!EnsureCallable(*input->Child(TGenReadTable::idx_Table), ctx)) { return TStatus::Error; } @@ -157,9 +171,21 @@ namespace NYql { } } + // Determine cluster name TString clusterName{input->Child(TGenReadTable::idx_DataSource)->Child(1)->Content()}; - TString tableName{input->Child(TGenReadTable::idx_Table)->Content()}; + // Determine table name + const auto tableNode = input->Child(TGenReadTable::idx_Table); + if (!TGenTable::Match(tableNode)) { + ctx.AddError(TIssue(ctx.GetPosition(tableNode->Pos()), + TStringBuilder() << "Expected " << TGenTable::CallableName())); + return TStatus::Error; + } + + TGenTable table(tableNode); + const auto tableName = table.Name().StringValue(); + + // Extract table metadata auto [tableMeta, issue] = State_->GetTable(clusterName, tableName, ctx.GetPosition(input->Pos())); if (issue.has_value()) { ctx.AddError(issue.value()); diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp index c36536b6fd17..bb5b042b6734 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp @@ -91,7 +91,7 @@ namespace NYql { .Input() .World(genReadTable.World()) .Cluster(genReadTable.DataSource().Cluster()) - .Table(genReadTable.Table()) + .Table(genReadTable.Table().Name()) .Token() .Name().Build(token) .Build() diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp index 71d3e977362a..d243d597e560 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp @@ -25,16 +25,17 @@ namespace NYql { using namespace NKikimr; using namespace NKikimr::NMiniKQL; - struct TGenericTableDescription { - using TPtr = std::shared_ptr; - - NYql::TGenericDataSourceInstance DataSourceInstance; - std::optional Response; - }; class TGenericLoadTableMetadataTransformer: public TGraphTransformerBase { + struct TTableDescription { + using TPtr = std::shared_ptr; + + NYql::TGenericDataSourceInstance DataSourceInstance; + std::optional Response; + }; + using TMapType = - std::unordered_map>; + std::unordered_map>; public: TGenericLoadTableMetadataTransformer(TGenericState::TPtr state) @@ -101,7 +102,7 @@ namespace NYql { handles.emplace_back(promise.GetFuture()); // preserve data source instance for the further usage - auto emplaceIt = Results_.emplace(std::make_pair(item, std::make_shared())); + auto emplaceIt = Results_.emplace(std::make_pair(item, std::make_shared())); auto desc = emplaceIt.first->second; desc->DataSourceInstance = request.data_source_instance(); @@ -171,6 +172,7 @@ namespace NYql { auto row = Build(ctx, read.Pos()) .Name("row") .Done(); + auto emptyPredicate = Build(ctx, read.Pos()) .Args({row}) .Body() @@ -178,10 +180,14 @@ namespace NYql { .Build() .Done().Ptr(); + auto table = Build(ctx, read.Pos()) + .Name().Value(tableName).Build() + .Splits().Build().Done(); + ins.first->second = Build(ctx, read.Pos()) .World(read.World()) .DataSource(read.DataSource()) - .Table().Value(tableName).Build() + .Table(table) .Columns().Build() .FilterPredicate(emptyPredicate) .Done().Ptr(); diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_physical_opt.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_physical_opt.cpp index 73e0101dab07..cce7d7dd5f6f 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_physical_opt.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_physical_opt.cpp @@ -72,7 +72,7 @@ namespace NYql { // Get table metadata const auto [tableMeta, issue] = State_->GetTable( read.DataSource().Cluster().Value(), - read.Table().Value(), + read.Table().Name().Value(), ctx.GetPosition(node.Pos())); if (issue.has_value()) { ctx.AddError(issue.value());