From 7bdce5eed4f79b42e7a96d9f455cf957687faf78 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Tue, 13 Feb 2024 14:37:36 +0000 Subject: [PATCH 1/3] init --- ydb/library/yql/ast/yql_expr.h | 21 ++++++++++++++++ ydb/library/yql/core/type_ann/type_ann_pg.cpp | 22 ++++++++-------- .../yql/parser/pg_wrapper/comp_factory.cpp | 25 +++++++++++++------ ydb/library/yql/sql/pg/pg_sql.cpp | 23 ++++++++++++----- .../sql/dq_file/part12/canondata/result.json | 6 ++--- .../hybrid_file/part8/canondata/result.json | 6 ++--- .../tests/sql/sql2yql/canondata/result.json | 6 ++--- .../tests/sql/suites/pg_catalog/pg_class.sql | 4 ++- .../part12/canondata/result.json | 12 ++++----- 9 files changed, 84 insertions(+), 41 deletions(-) diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index 21329b8ab0a2..3ce03fe1df25 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -531,6 +532,26 @@ class TStructExprType : public TTypeAnnotationNode { return it - Items.begin(); } + TMaybe FindItemI(const TStringBuf& name) const { + auto strict = FindItem(name); + if (strict) { + return strict; + } + + TMaybe ret; + for (ui32 i = 0; i < Items.size(); ++i) { + if (AsciiEqualsIgnoreCase(name, Items[i]->GetName())) { + if (ret) { + return Nothing(); + } + + ret = i; + } + } + + return ret; + } + const TTypeAnnotationNode* FindItemType(const TStringBuf& name) const { const auto it = LowerBound(Items.begin(), Items.end(), name, TItemLess()); if (it == Items.end() || (*it)->GetName() != name) { diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp index c0960439516c..c28736d2928b 100644 --- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp @@ -151,7 +151,7 @@ IGraphTransformer::TStatus InferPgCommonType(TPositionHandle pos, const TExprNod { size_t j = 0; for (const auto& col : *childColumnOrder) { - auto itemIdx = structType->FindItem(col); + auto itemIdx = structType->FindItemI(col); YQL_ENSURE(itemIdx); const auto* type = structType->GetItems()[*itemIdx]->GetItemType(); @@ -1577,7 +1577,7 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet& } } - auto pos = x.Type->FindItem(node->Tail().Content()); + auto pos = x.Type->FindItemI(node->Tail().Content()); if (pos) { foundAlias = x.Alias; ++matches; @@ -1589,7 +1589,7 @@ bool ScanColumns(TExprNode::TPtr root, TInputs& inputs, const THashSet& } if (x.Priority == TInput::External) { - x.UsedExternalColumns.insert(TString(node->Tail().Content())); + x.UsedExternalColumns.insert(TString(x.Type->GetItems()[*pos]->GetName())); } } } @@ -1780,12 +1780,12 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSetFindItem(ref); + auto pos = x.Type->FindItemI(ref); if (pos) { auto item = x.Type->GetItems()[*pos]; item = AddAlias(x.Alias, item, ctx); items.push_back(item); - usedRefs.insert(ref); + usedRefs.insert(TString(item->GetName())); } } @@ -1795,7 +1795,7 @@ void AddColumns(const TInputs& inputs, const bool* hasStar, const THashSetfind(x.Alias)->second) { - auto pos = x.Type->FindItem(ref); + auto pos = x.Type->FindItemI(ref); if (pos) { auto item = x.Type->GetItems()[*pos]; item = AddAlias(x.Alias, item, ctx); @@ -1882,12 +1882,12 @@ IGraphTransformer::TStatus RebuildLambdaColumns(const TExprNode::TPtr& root, con } } - auto pos = x.Type->FindItem(node->Tail().Content()); + auto pos = x.Type->FindItemI(node->Tail().Content()); if (pos) { return ctx.Expr.Builder(node->Pos()) .Callable("Member") .Add(0, argNode) - .Atom(1, MakeAliasedColumn(x.Alias, node->Tail().Content())) + .Atom(1, MakeAliasedColumn(x.Alias, x.Type->GetItems()[*pos]->GetName())) .Seal() .Build(); } @@ -2754,7 +2754,7 @@ bool GatherExtraSortColumns(const TExprNode& data, const TInputs& inputs, TExprN continue; } - auto pos = x.Type->FindItem(column); + auto pos = x.Type->FindItemI(column); if (pos) { index = inputIndex; break; @@ -3396,7 +3396,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN TVector newStructItems; TColumnOrder newOrder; for (ui32 i = 0; i < p->Child(2)->ChildrenSize(); ++i) { - auto pos = inputStructType->FindItem((*columnOrder)[i]); + auto pos = inputStructType->FindItemI((*columnOrder)[i]); YQL_ENSURE(pos); auto type = inputStructType->GetItems()[*pos]->GetItemType(); newOrder.push_back(TString(p->Child(2)->Child(i)->Content())); @@ -4147,7 +4147,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN const auto type = data.Child(j)->Tail().GetTypeAnn()->Cast()-> GetType()->Cast(); for (const auto& col : x.UsedExternalColumns) { - auto pos = type->FindItem(col); + auto pos = type->FindItemI(col); YQL_ENSURE(pos); items.push_back(type->GetItems()[*pos]); } diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 930a9ed9e76a..a853923ba5ce 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -411,12 +411,13 @@ class TPgTableContent : public TMutableComputationNode { ApplyFillers(AllPgDatabaseStatFillers, Y_ARRAY_SIZE(AllPgDatabaseStatFillers), PgDatabaseStatFillers_); } else if (Table_ == "pg_class") { static const std::pair AllPgClassFillers[] = { - {"oid", [](const NPg::TTableInfo& desc, ui32) { return ScalarDatumToPod(ObjectIdGetDatum(desc.Oid)); }}, - {"relispartition", [](const NPg::TTableInfo&, ui32) { return ScalarDatumToPod(BoolGetDatum(false)); }}, - {"relkind", [](const NPg::TTableInfo& desc, ui32) { return ScalarDatumToPod(CharGetDatum(desc.Kind)); }}, - {"relname", [](const NPg::TTableInfo& desc, ui32) { return PointerDatumToPod((Datum)MakeFixedString(desc.Name, NAMEDATALEN)); }}, - {"relnamespace", [](const NPg::TTableInfo&, ui32 namespaceOid) { return ScalarDatumToPod(ObjectIdGetDatum(namespaceOid)); }}, - {"relowner", [](const NPg::TTableInfo&, ui32) { return ScalarDatumToPod(ObjectIdGetDatum(1)); }}, + {"oid", [](const NPg::TTableInfo& desc, ui32, ui32) { return ScalarDatumToPod(ObjectIdGetDatum(desc.Oid)); }}, + {"relispartition", [](const NPg::TTableInfo&, ui32, ui32) { return ScalarDatumToPod(BoolGetDatum(false)); }}, + {"relkind", [](const NPg::TTableInfo& desc, ui32, ui32) { return ScalarDatumToPod(CharGetDatum(desc.Kind)); }}, + {"relname", [](const NPg::TTableInfo& desc, ui32, ui32) { return PointerDatumToPod((Datum)MakeFixedString(desc.Name, NAMEDATALEN)); }}, + {"relnamespace", [](const NPg::TTableInfo&, ui32 namespaceOid,ui32) { return ScalarDatumToPod(ObjectIdGetDatum(namespaceOid)); }}, + {"relowner", [](const NPg::TTableInfo&, ui32, ui32) { return ScalarDatumToPod(ObjectIdGetDatum(1)); }}, + {"relam", [](const NPg::TTableInfo&, ui32, ui32 amOid) { return ScalarDatumToPod(ObjectIdGetDatum(amOid)); }}, }; ApplyFillers(AllPgClassFillers, Y_ARRAY_SIZE(AllPgClassFillers), PgClassFillers_); @@ -650,11 +651,19 @@ class TPgTableContent : public TMutableComputationNode { namespaces[desc.Name] = oid; }); + ui32 btreeAmOid = 0; + NPg::EnumAm([&](ui32 oid, const NPg::TAmDesc& desc) { + if (desc.AmName == "btree") { + btreeAmOid = oid; + } + }); + for (const auto& t : tables) { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgClassFillers_.size(), items); for (ui32 i = 0; i < PgClassFillers_.size(); ++i) { - items[i] = PgClassFillers_[i](t, namespaces[t.Schema]); + ui32 amOid = (t.Kind == NPg::ERelKind::Relation) ? btreeAmOid : 0; + items[i] = PgClassFillers_[i](t, namespaces[t.Schema], amOid); } rows.emplace_back(row); @@ -735,7 +744,7 @@ class TPgTableContent : public TMutableComputationNode { using TColumnsFiller = NUdf::TUnboxedValuePod(*)(const NPg::TColumnInfo&); TVector ColumnsFillers_; - using TPgClassFiller = NUdf::TUnboxedValuePod(*)(const NPg::TTableInfo&, ui32 namespaceOid); + using TPgClassFiller = NUdf::TUnboxedValuePod(*)(const NPg::TTableInfo&, ui32 namespaceOid, ui32 amOid); TVector PgClassFillers_; }; diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 3caaced06702..1c05938c6652 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -142,6 +142,10 @@ int StrCompare(const char* s1, const char* s2) { return strcmp(s1 ? s1 : "", s2 ? s2 : ""); } +int StrICompare(const char* s1, const char* s2) { + return stricmp(s1 ? s1 : "", s2 ? s2 : ""); +} + std::shared_ptr ListMake1(void* cell) { return std::shared_ptr(list_make1(cell), list_free); } @@ -2104,7 +2108,7 @@ class TConverter : public IPGParseEvents { AddError(TStringBuilder() << "VariableSetStmt, expected string literal for " << value->name << " option"); return nullptr; } - TString rawStr = TString(StrVal(val)); + TString rawStr = to_lower(TString(StrVal(val))); if (name != "search_path") { AddError(TStringBuilder() << "VariableSetStmt, set_config doesn't support that option:" << name); return nullptr; @@ -2561,20 +2565,27 @@ class TConverter : public IPGParseEvents { TAstNode* BuildClusterSinkOrSourceExpression( bool isSink, const TStringBuf schemaname) { - const auto p = Settings.ClusterMapping.FindPtr(schemaname); + TString usedCluster(schemaname); + auto p = Settings.ClusterMapping.FindPtr(usedCluster); + if (!p) { + usedCluster = to_lower(usedCluster); + p = Settings.ClusterMapping.FindPtr(usedCluster); + } + if (!p) { AddError(TStringBuilder() << "Unknown cluster: " << schemaname); return nullptr; } - return L(isSink ? A("DataSink") : A("DataSource"), QAX(*p), QAX(schemaname.Data())); + return L(isSink ? A("DataSink") : A("DataSource"), QAX(*p), QAX(usedCluster)); } TAstNode* BuildTableKeyExpression(const TStringBuf relname, const TStringBuf cluster, bool isScheme = false ) { - bool noPrefix = (cluster == "pg_catalog" || cluster == "information_schema"); - TString tableName = noPrefix ? TString(relname) : TablePathPrefix + relname; + auto lowerCluster = to_lower(TString(cluster)); + bool noPrefix = (lowerCluster == "pg_catalog" || lowerCluster == "information_schema"); + TString tableName = noPrefix ? to_lower(TString(relname)) : TablePathPrefix + relname; return L(A("Key"), QL(QA(isScheme ? "tablescheme" : "table"), L(A("String"), QAX(std::move(tableName))))); } @@ -3520,7 +3531,7 @@ class TConverter : public IPGParseEvents { !typeName->pct_type && (ListLength(typeName->names) == 2 && NodeTag(ListNodeNth(typeName->names, 0)) == T_String && - !StrCompare(StrVal(ListNodeNth(typeName->names, 0)), "pg_catalog") || ListLength(typeName->names) == 1) && + !StrICompare(StrVal(ListNodeNth(typeName->names, 0)), "pg_catalog") || ListLength(typeName->names) == 1) && NodeTag(ListNodeNth(typeName->names, ListLength(typeName->names) - 1)) == T_String; if (NodeTag(arg) == T_A_Const && diff --git a/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json index fcb85e767b6a..546bb33a091d 100644 --- a/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json @@ -2515,9 +2515,9 @@ ], "test.test[pg_catalog-pg_class-default.txt-Debug]": [ { - "checksum": "ec01c3e80fa9ed400a36e0e984f826bd", - "size": 7046, - "uri": "https://{canondata_backend}/1936997/15647165297f31fc500aaeb8506af785801000d5/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql_patched" + "checksum": "b798d692cdb0050db23e0af30a0bf365", + "size": 7854, + "uri": "https://{canondata_backend}/1936273/86cc69b84d2bd29ced3a8946b9d79484a91fffaa/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_class-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json index e4328e3c83da..e7d043b17fbc 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json @@ -2381,9 +2381,9 @@ ], "test.test[pg_catalog-pg_class-default.txt-Debug]": [ { - "checksum": "fb682e3f79671dbce0b6f7bdea376718", - "size": 7045, - "uri": "https://{canondata_backend}/1781765/9c2a131db7c99a7c31942aabf8738f3bac969d91/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql_patched" + "checksum": "afa755b11b36a96a2cecf6f10b7ee834", + "size": 7853, + "uri": "https://{canondata_backend}/1880306/9b3d48683fc0ab2f3a465a16fcec6b6f0793c208/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_class-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 61bdac4c8ea2..1706c5faddc4 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -13315,9 +13315,9 @@ ], "test_sql2yql.test[pg_catalog-pg_class]": [ { - "checksum": "714f0205f14cecf109480cb2fc163a16", - "size": 1809, - "uri": "https://{canondata_backend}/1777230/0c388884f13a4e27fa979282c1cb1b2af98b6a1e/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_class_/sql.yql" + "checksum": "1a29aea58de89c752df9fbb263f12260", + "size": 1983, + "uri": "https://{canondata_backend}/1942173/1a97c7aac68c22baf8625fe091ac9f4c169b3dd4/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_class_/sql.yql" } ], "test_sql2yql.test[pg_catalog-pg_database]": [ diff --git a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_class.sql b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_class.sql index e3be64c5f4b2..28f3d66e2823 100644 --- a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_class.sql +++ b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_class.sql @@ -12,7 +12,9 @@ max(relispartition::text) max_isrel, max(relkind) max_relkind, max(relname) max_relname, max(relnamespace) max_relns, -max(relowner) max_relowner +max(relowner) max_relowner, +min(relam) min_am, +max(relam) max_am from pg_catalog.pg_class; diff --git a/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json index b9877b48d940..7dcb370e96ca 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json @@ -2348,9 +2348,9 @@ ], "test.test[pg_catalog-pg_class-default.txt-Debug]": [ { - "checksum": "cae74a693d7a4e3f1578f754adcae5c6", - "size": 6971, - "uri": "https://{canondata_backend}/1871182/f42651a4c8505a845b01023d9d7e8e03f2e50c2d/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql" + "checksum": "99f6276f6794cfca07f55964603dacfe", + "size": 7779, + "uri": "https://{canondata_backend}/1937492/1e7a7859528b242167c1536c8c76d6682b16c41b/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Debug_/opt.yql" } ], "test.test[pg_catalog-pg_class-default.txt-Plan]": [ @@ -2362,9 +2362,9 @@ ], "test.test[pg_catalog-pg_class-default.txt-Results]": [ { - "checksum": "878ba76e165d869792a16cc18e29da6e", - "size": 4209, - "uri": "https://{canondata_backend}/1871182/f42651a4c8505a845b01023d9d7e8e03f2e50c2d/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Results_/results.txt" + "checksum": "b706bba317a2352e9f76e66be6443028", + "size": 4787, + "uri": "https://{canondata_backend}/1937492/1e7a7859528b242167c1536c8c76d6682b16c41b/resource.tar.gz#test.test_pg_catalog-pg_class-default.txt-Results_/results.txt" } ], "test.test[pg_catalog-pg_description_pg_syntax-default.txt-Debug]": [ From d0efa0f9e43bcc517d10844a8e0f2696b3c3b3dc Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Tue, 13 Feb 2024 16:28:04 +0000 Subject: [PATCH 2/3] more fixes --- ydb/library/yql/ast/yql_expr.cpp | 2 +- ydb/library/yql/parser/pg_wrapper/comp_factory.cpp | 4 ++++ .../tests/sql/dq_file/part0/canondata/result.json | 6 +++--- .../tests/sql/dq_file/part1/canondata/result.json | 6 +++--- .../sql/hybrid_file/part4/canondata/result.json | 6 +++--- .../sql/hybrid_file/part8/canondata/result.json | 6 +++--- .../yql/tests/sql/sql2yql/canondata/result.json | 12 ++++++------ .../yql/tests/sql/suites/pg_catalog/pg_database.sql | 5 ++++- .../sql/suites/pg_catalog/pg_namespace_pg_syntax.sql | 2 +- .../sql/yt_native_file/part0/canondata/result.json | 12 ++++++------ .../sql/yt_native_file/part1/canondata/result.json | 12 ++++++------ 11 files changed, 40 insertions(+), 33 deletions(-) diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 4c74101b2ea8..0cc8aac25167 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -3206,7 +3206,7 @@ ui32 TPgExprType::GetFlags(ui32 typeId) { const auto& desc = *descPtr; ui32 ret = TypeHasManyValues | TypeHasOptional; - if (!desc.SendFuncId || !desc.ReceiveFuncId) { + if ((!desc.SendFuncId || !desc.ReceiveFuncId) && (!desc.OutFuncId || !desc.InFuncId)) { ret |= TypeNonPersistable; } diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index a853923ba5ce..e70b9c991b4d 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -312,6 +312,9 @@ class TPgTableContent : public TMutableComputationNode { {"datname", [](ui32 index) { return PointerDatumToPod((Datum)(MakeFixedString( index == 1 ? "template1" : (index == 2 ? "template0" : "postgres"), NAMEDATALEN))); }}, {"encoding", [](ui32) { return ScalarDatumToPod(Int32GetDatum(PG_UTF8)); }}, + {"datacl", [](ui32) { return NUdf::TUnboxedValuePod(); }}, + {"datcollate", [](ui32) { return PointerDatumToPod((Datum)(MakeFixedString("C", NAMEDATALEN))); }}, + {"datctype", [](ui32) { return PointerDatumToPod((Datum)(MakeFixedString("C", NAMEDATALEN))); }}, }; ApplyFillers(AllPgDatabaseFillers, Y_ARRAY_SIZE(AllPgDatabaseFillers), PgDatabaseFillers_); @@ -346,6 +349,7 @@ class TPgTableContent : public TMutableComputationNode { static const std::pair AllPgNamespaceFillers[] = { {"nspname", [](const NPg::TNamespaceDesc& desc) {return PointerDatumToPod((Datum)MakeFixedString(desc.Name, NAMEDATALEN));}}, {"oid", [](const NPg::TNamespaceDesc& desc) { return ScalarDatumToPod(ObjectIdGetDatum(desc.Oid)); }}, + {"nspowner", [](const NPg::TNamespaceDesc&) { return ScalarDatumToPod(ObjectIdGetDatum(1)); }}, }; ApplyFillers(AllPgNamespaceFillers, Y_ARRAY_SIZE(AllPgNamespaceFillers), PgNamespaceFillers_); diff --git a/ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json index e312900a8c99..b850f52fd3a6 100644 --- a/ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json @@ -2490,9 +2490,9 @@ ], "test.test[pg_catalog-pg_database-default.txt-Debug]": [ { - "checksum": "71ee0fd1c96cd90fb8c22d4cc462cb36", - "size": 546, - "uri": "https://{canondata_backend}/1931696/c91b6a3236adb31c0c0b201a2986571c6193e309/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql_patched" + "checksum": "20c4e3d0bd69a09eecc8a91e4c1e8cee", + "size": 1136, + "uri": "https://{canondata_backend}/1784826/d2a1b732d518bb4ef49d545b2ddf9c004a5d5c75/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_database-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json index ff661fc9143e..d8dbe5273792 100644 --- a/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json @@ -2315,9 +2315,9 @@ ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Debug]": [ { - "checksum": "cb066fc3c8e6e0d939fdf7e754fa5afe", - "size": 492, - "uri": "https://{canondata_backend}/1942525/8c1f4807d6d4556cc05ec1ea8c94dec50e30ed6b/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql_patched" + "checksum": "c48f9e238b15f33f292ef2eb4a5176ef", + "size": 546, + "uri": "https://{canondata_backend}/1937027/3d8ae7405c87d4dd8f5110c9cafc9e3ec447e435/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json index 8ffa271e7e6f..945a04062abb 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json @@ -2269,9 +2269,9 @@ ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Debug]": [ { - "checksum": "4ea26156335e842d2ff293406cf5a5cd", - "size": 491, - "uri": "https://{canondata_backend}/1889210/431569691fa60b20bf9ef4cc94610d8f1b1518e2/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql_patched" + "checksum": "e1950d6a40e258953cb9c7f36d5bb336", + "size": 545, + "uri": "https://{canondata_backend}/1937027/8b932d9e8daf49fb9e777a1b6fa53c785126e3d6/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json index e7d043b17fbc..9e95ffc3b2f9 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json @@ -2395,9 +2395,9 @@ ], "test.test[pg_catalog-pg_database-default.txt-Debug]": [ { - "checksum": "0f16f04a97c8114d94db8c9ac5da067e", - "size": 545, - "uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql_patched" + "checksum": "303b99c5f2728ec6b79e41273eeb486d", + "size": 1135, + "uri": "https://{canondata_backend}/1923547/9239cf6dc7870b94856822425e617d80d75f9a89/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg_catalog-pg_database-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 1706c5faddc4..6d5c1963d954 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -13322,9 +13322,9 @@ ], "test_sql2yql.test[pg_catalog-pg_database]": [ { - "checksum": "3c66358cd738a51ccd0766e18e21fc45", - "size": 1077, - "uri": "https://{canondata_backend}/1942525/b1770d18f4fe73123c8ba7b8888beae32aa4e70b/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_database_/sql.yql" + "checksum": "5016020047c31d0a12c29d7b6fdd4b1a", + "size": 1319, + "uri": "https://{canondata_backend}/1936842/abd9f49421ab39ca2bf1b40d9a65f1c24f457928/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_database_/sql.yql" } ], "test_sql2yql.test[pg_catalog-pg_description_pg_syntax]": [ @@ -13350,9 +13350,9 @@ ], "test_sql2yql.test[pg_catalog-pg_namespace_pg_syntax]": [ { - "checksum": "e7ca24866a5066576dfab540799aef83", - "size": 773, - "uri": "https://{canondata_backend}/1871002/9140ba5c900f07690dd3e7f43001e6ff24bdb036/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_namespace_pg_syntax_/sql.yql" + "checksum": "9e4c757133b4a0784952c6c967d504bb", + "size": 846, + "uri": "https://{canondata_backend}/1600758/c8681bdb1ea76420d75d8e7d6d355f3e5abf1d95/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_namespace_pg_syntax_/sql.yql" } ], "test_sql2yql.test[pg_catalog-pg_roles]": [ diff --git a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_database.sql b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_database.sql index 309b0b1f581b..4b4d7c7c9af0 100644 --- a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_database.sql +++ b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_database.sql @@ -5,7 +5,10 @@ datdba, datistemplate, datname, encoding, -oid +oid, +datacl is null, +datcollate, +datctype from pg_catalog.pg_database order by oid diff --git a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_namespace_pg_syntax.sql b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_namespace_pg_syntax.sql index bbd708431cbc..89e355f157af 100644 --- a/ydb/library/yql/tests/sql/suites/pg_catalog/pg_namespace_pg_syntax.sql +++ b/ydb/library/yql/tests/sql/suites/pg_catalog/pg_namespace_pg_syntax.sql @@ -1,3 +1,3 @@ --!syntax_pg -select nspname,oid from pg_catalog.pg_namespace order by oid; +select nspname,oid,nspowner from pg_catalog.pg_namespace order by oid; diff --git a/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json index 5b15951f97af..48f66761b74c 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json @@ -2206,9 +2206,9 @@ ], "test.test[pg_catalog-pg_database-default.txt-Debug]": [ { - "checksum": "17468616f194e97d0c9ee189a54168d2", - "size": 488, - "uri": "https://{canondata_backend}/1903885/9f9ffd6205a460d0ccfa5664f67266cef944c50f/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql" + "checksum": "02d473bd3ab0737683dbb8c27431e87f", + "size": 1068, + "uri": "https://{canondata_backend}/1784826/565b8e27aed424ae566a2615fc8b9f1d8a304a47/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Debug_/opt.yql" } ], "test.test[pg_catalog-pg_database-default.txt-Plan]": [ @@ -2220,9 +2220,9 @@ ], "test.test[pg_catalog-pg_database-default.txt-Results]": [ { - "checksum": "66853b7df1c55be98264ba78cf15d434", - "size": 2596, - "uri": "https://{canondata_backend}/1942671/52a125a1bb7634ddba0f986b6cf8177f5b9c01df/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Results_/results.txt" + "checksum": "689293b6c55e4ec37a47713fbb92162f", + "size": 3653, + "uri": "https://{canondata_backend}/1784826/565b8e27aed424ae566a2615fc8b9f1d8a304a47/resource.tar.gz#test.test_pg_catalog-pg_database-default.txt-Results_/results.txt" } ], "test.test[pg_catalog-pg_stat_gssapi-default.txt-Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json index 9be4adadb192..1dd37ea37107 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json @@ -2120,9 +2120,9 @@ ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Debug]": [ { - "checksum": "64e0686c6e328d97fe9706089939fe73", - "size": 434, - "uri": "https://{canondata_backend}/1784826/f83822f9d1d0e50a7c1305c00312ff353e475b1c/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql" + "checksum": "b83592d4f3ea974bb0c4dbd4064e891b", + "size": 486, + "uri": "https://{canondata_backend}/1937027/388f52e8b86076faef605e3873b79f2ccfab1169/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Debug_/opt.yql" } ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Plan]": [ @@ -2134,9 +2134,9 @@ ], "test.test[pg_catalog-pg_namespace_pg_syntax-default.txt-Results]": [ { - "checksum": "2a36b50f473e6fcf40a3a7cf48109f6f", - "size": 1186, - "uri": "https://{canondata_backend}/1900335/8f9b5c6c5c8ab606e213d4fcbbba4bb451f86310/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Results_/results.txt" + "checksum": "a6cba2095213c0a8ea17c0e31a5718e0", + "size": 1534, + "uri": "https://{canondata_backend}/1937027/388f52e8b86076faef605e3873b79f2ccfab1169/resource.tar.gz#test.test_pg_catalog-pg_namespace_pg_syntax-default.txt-Results_/results.txt" } ], "test.test[produce-process_rows_and_filter--Debug]": [ From 9a89db9accca8eb2fc7af6ed3f6f5d4bcc4ce8f6 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Tue, 13 Feb 2024 16:52:09 +0000 Subject: [PATCH 3/3] simplify nulls --- .../yql/parser/pg_wrapper/comp_factory.cpp | 85 +++++++++++++------ 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index e70b9c991b4d..7ed88b35ccb5 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -312,7 +312,6 @@ class TPgTableContent : public TMutableComputationNode { {"datname", [](ui32 index) { return PointerDatumToPod((Datum)(MakeFixedString( index == 1 ? "template1" : (index == 2 ? "template0" : "postgres"), NAMEDATALEN))); }}, {"encoding", [](ui32) { return ScalarDatumToPod(Int32GetDatum(PG_UTF8)); }}, - {"datacl", [](ui32) { return NUdf::TUnboxedValuePod(); }}, {"datcollate", [](ui32) { return PointerDatumToPod((Datum)(MakeFixedString("C", NAMEDATALEN))); }}, {"datctype", [](ui32) { return PointerDatumToPod((Datum)(MakeFixedString("C", NAMEDATALEN))); }}, }; @@ -456,10 +455,6 @@ class TPgTableContent : public TMutableComputationNode { fillers[*pos] = func; } } - - for (const auto& f : fillers) { - Y_ENSURE(f); - } } NUdf::TUnboxedValuePod DoCalculate(TComputationContext& compCtx) const { @@ -474,7 +469,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgTypeFillers_.size(), items); for (ui32 i = 0; i < PgTypeFillers_.size(); ++i) { - items[i] = PgTypeFillers_[i](desc); + if (PgTypeFillers_[i]) { + items[i] = PgTypeFillers_[i](desc); + } } rows.emplace_back(row); @@ -484,7 +481,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgDatabaseFillers_.size(), items); for (ui32 i = 0; i < PgDatabaseFillers_.size(); ++i) { - items[i] = PgDatabaseFillers_[i](index); + if (PgDatabaseFillers_[i]) { + items[i] = PgDatabaseFillers_[i](index); + } } rows.emplace_back(row); @@ -494,7 +493,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgTablespaceFillers_.size(), items); for (ui32 i = 0; i < PgTablespaceFillers_.size(); ++i) { - items[i] = PgTablespaceFillers_[i](index); + if (PgTablespaceFillers_[i]) { + items[i] = PgTablespaceFillers_[i](index); + } } rows.emplace_back(row); @@ -504,7 +505,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgShDescriptionFillers_.size(), items); for (ui32 i = 0; i < PgShDescriptionFillers_.size(); ++i) { - items[i] = PgShDescriptionFillers_[i](index); + if (PgShDescriptionFillers_[i]) { + items[i] = PgShDescriptionFillers_[i](index); + } } rows.emplace_back(row); @@ -513,7 +516,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgStatGssapiFillers_.size(), items); for (ui32 i = 0; i < PgStatGssapiFillers_.size(); ++i) { - items[i] = PgStatGssapiFillers_[i](); + if (PgStatGssapiFillers_[i]) { + items[i] = PgStatGssapiFillers_[i](); + } } rows.emplace_back(row); @@ -522,7 +527,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(ItemType_->GetMembersCount(), items); for (ui32 i = 0; i < PgNamespaceFillers_.size(); ++i) { - items[i] = PgNamespaceFillers_[i](desc); + if (PgNamespaceFillers_[i]) { + items[i] = PgNamespaceFillers_[i](desc); + } } rows.emplace_back(row); @@ -531,8 +538,10 @@ class TPgTableContent : public TMutableComputationNode { NPg::EnumAm([&](ui32 oid, const NPg::TAmDesc& desc) { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(ItemType_->GetMembersCount(), items); - for (ui32 i = 0; i < ItemType_->GetMembersCount(); ++i) { - items[i] = PgAmFillers_[i](desc); + for (ui32 i = 0; i < PgAmFillers_.size(); ++i) { + if (PgAmFillers_[i]) { + items[i] = PgAmFillers_[i](desc); + } } rows.emplace_back(row); @@ -546,7 +555,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = oid; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -559,7 +570,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = oid; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -572,7 +585,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = oid; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -586,7 +601,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = desc_.ConversionId; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -600,7 +617,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = desc_.OperId; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -614,7 +633,9 @@ class TPgTableContent : public TMutableComputationNode { for (ui32 i = 0; i < PgDescriptionFillers_.size(); ++i) { desc.Objoid = desc_.ProcId; desc.Description = desc_.Descr; - items[i] = PgDescriptionFillers_[i](desc); + if (PgDescriptionFillers_[i]) { + items[i] = PgDescriptionFillers_[i](desc); + } } rows.emplace_back(row); @@ -625,7 +646,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgTablesFillers_.size(), items); for (ui32 i = 0; i < PgTablesFillers_.size(); ++i) { - items[i] = PgTablesFillers_[i](t); + if (PgTablesFillers_[i]) { + items[i] = PgTablesFillers_[i](t); + } } rows.emplace_back(row); @@ -634,7 +657,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgRolesFillers_.size(), items); for (ui32 i = 0; i < PgRolesFillers_.size(); ++i) { - items[i] = PgRolesFillers_[i](); + if (PgRolesFillers_[i]) { + items[i] = PgRolesFillers_[i](); + } } rows.emplace_back(row); @@ -643,7 +668,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgDatabaseStatFillers_.size(), items); for (ui32 i = 0; i < PgDatabaseStatFillers_.size(); ++i) { - items[i] = PgDatabaseStatFillers_[i](index); + if (PgDatabaseStatFillers_[i]) { + items[i] = PgDatabaseStatFillers_[i](index); + } } rows.emplace_back(row); @@ -663,11 +690,13 @@ class TPgTableContent : public TMutableComputationNode { }); for (const auto& t : tables) { + const ui32 amOid = (t.Kind == NPg::ERelKind::Relation) ? btreeAmOid : 0; NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgClassFillers_.size(), items); for (ui32 i = 0; i < PgClassFillers_.size(); ++i) { - ui32 amOid = (t.Kind == NPg::ERelKind::Relation) ? btreeAmOid : 0; - items[i] = PgClassFillers_[i](t, namespaces[t.Schema], amOid); + if (PgClassFillers_[i]) { + items[i] = PgClassFillers_[i](t, namespaces[t.Schema], amOid); + } } rows.emplace_back(row); @@ -680,7 +709,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(TablesFillers_.size(), items); for (ui32 i = 0; i < TablesFillers_.size(); ++i) { - items[i] = TablesFillers_[i](t); + if (TablesFillers_[i]) { + items[i] = TablesFillers_[i](t); + } } rows.emplace_back(row); @@ -692,7 +723,9 @@ class TPgTableContent : public TMutableComputationNode { NUdf::TUnboxedValue* items; auto row = compCtx.HolderFactory.CreateDirectArrayHolder(ColumnsFillers_.size(), items); for (ui32 i = 0; i < ColumnsFillers_.size(); ++i) { - items[i] = ColumnsFillers_[i](c); + if (ColumnsFillers_[i]) { + items[i] = ColumnsFillers_[i](c); + } } rows.emplace_back(row);