From ed7c2076ad301707f7f6732cb699c29f992ce931 Mon Sep 17 00:00:00 2001 From: Roman Udovichenko Date: Thu, 31 Oct 2024 17:42:16 +0300 Subject: [PATCH] [yt] Mixed gateway (#11146) --- .../providers/yt/gateway/file/yql_yt_file.cpp | 107 +++++- .../yql/providers/yt/gateway/mixed/ya.make | 22 ++ .../yt/gateway/mixed/yql_yt_mixed.cpp | 323 ++++++++++++++++++ .../providers/yt/gateway/mixed/yql_yt_mixed.h | 10 + .../yt/gateway/native/yql_yt_native.cpp | 127 +++++++ .../yt/gateway/profile/yql_yt_profiling.cpp | 12 + .../qplayer/yql_yt_qplayer_gateway.cpp | 20 +- ydb/library/yql/providers/yt/gateway/ya.make | 1 + .../providers/yt/provider/yql_yt_gateway.h | 46 +++ .../part9/canondata/result.json | 8 +- .../part1/canondata/result.json | 18 +- .../part11/canondata/result.json | 30 +- .../part13/canondata/result.json | 14 +- .../part16/canondata/result.json | 28 +- .../part17/canondata/result.json | 6 +- .../part18/canondata/result.json | 14 +- .../part19/canondata/result.json | 4 +- .../part8/canondata/result.json | 24 +- .../part9/canondata/result.json | 4 +- 19 files changed, 732 insertions(+), 86 deletions(-) create mode 100644 ydb/library/yql/providers/yt/gateway/mixed/ya.make create mode 100644 ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.cpp create mode 100644 ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.h diff --git a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp index fc5d6b101fae..de421a6130a9 100644 --- a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +++ b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -189,9 +190,11 @@ class TFileTransformProvider { const TString name(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); auto block = TUserDataStorage::FindUserDataBlock(UserDataBlocks, name); MKQL_ENSURE(block, "File not found: " << name); - MKQL_ENSURE(block->Type == EUserDataType::PATH, "FilePath not supported for non-file data block, name: " + MKQL_ENSURE(block->Type == EUserDataType::PATH || block->FrozenFile, "File is not frozen, name: " << name << ", block type: " << block->Type); - return TProgramBuilder(env, *Services->GetFunctionRegistry()).NewDataLiteral(block->Data); + return TProgramBuilder(env, *Services->GetFunctionRegistry()).NewDataLiteral( + block->Type == EUserDataType::PATH ? block->Data : block->FrozenFile->GetPath().GetPath() + ); }; } @@ -206,7 +209,7 @@ class TFileTransformProvider { continue; } - MKQL_ENSURE(x.second.Type == EUserDataType::PATH, "FilePath not supported for non-file data block, name: " + MKQL_ENSURE(x.second.Type == EUserDataType::PATH, "FolderPath not supported for non-file data block, name: " << x.first.Alias() << ", block type: " << x.second.Type); auto newFolderPath = x.second.Data.substr(0, x.second.Data.size() - (x.first.Alias().size() - folderName.size())); if (!folderPath) { @@ -234,9 +237,8 @@ class TFileTransformProvider { else if (block->Type == EUserDataType::RAW_INLINE_DATA) { return pgmBuilder.NewDataLiteral(block->Data); } - else if (Services->GetFileStorage() && block->Type == EUserDataType::URL) { - auto link = Services->GetFileStorage()->PutUrl(block->Data, ""); - auto content = TFileInput(link->GetPath()).ReadAll(); + else if (block->FrozenFile && block->Type == EUserDataType::URL) { + auto content = TFileInput(block->FrozenFile->GetPath().GetPath()).ReadAll(); return pgmBuilder.NewDataLiteral(content); } else { MKQL_ENSURE(false, "Unsupported block type"); @@ -336,6 +338,23 @@ class TFileTransformProvider { std::shared_ptr> ExtraArgs; }; +template +static inline TType OptionFromString(const TStringBuf value) { + if constexpr (std::is_same_v) { + return TString{value}; + } else if constexpr (std::is_same_v) { + return NYT::NodeFromYsonString(value); + } else { + return FromString(value); + } +} + +template +static inline const TType& NoOp(const TType& value) { + return value; +} + + /////////////////////////////////////////////////////////////////////////////////////////////////////// class TYtFileGateway : public IYtGateway { @@ -844,8 +863,12 @@ class TYtFileGateway : public IYtGateway { auto publish = TYtPublish(node); - auto mode = NYql::GetSetting(publish.Settings().Ref(), EYtSettingType::Mode); - bool append = mode && FromString(mode->Child(1)->Content()) == EYtWriteMode::Append; + EYtWriteMode mode = EYtWriteMode::Renew; + if (const auto modeSetting = NYql::GetSetting(publish.Settings().Ref(), EYtSettingType::Mode)) { + mode = FromString(modeSetting->Child(1)->Content()); + } + + bool append = mode == EYtWriteMode::Append; auto cluster = TString{publish.DataSink().Cluster().Value()}; bool isAnonymous = NYql::HasSetting(publish.Publish().Settings().Ref(), EYtSettingType::Anonymous); @@ -939,9 +962,65 @@ class TYtFileGateway : public IYtGateway { columnGroupsSpec = NYT::NodeFromYsonString(setting->Tail().Content()); } } - if (!append || !attrs.HasKey("schema") || !columnGroupsSpec.IsUndefined()) { + if (!append || !attrs.HasKey("schema") || !columnGroupsSpec.IsUndefined() || dstRowSpec->IsSorted()) { attrs["schema"] = RowSpecToYTSchema(spec[YqlRowSpecAttribute], nativeYtTypeCompatibility, columnGroupsSpec).ToNode(); } + + if (EYtWriteMode::Renew == mode || EYtWriteMode::RenewKeepMeta == mode) { + bool isTimestamp = false, isDuration = false; + TInstant stamp; + TDuration duration; + if (auto e = NYql::GetSetting(publish.Settings().Ref(), EYtSettingType::Expiration)) { + isDuration = TDuration::TryParse(e->Tail().Content(), duration); + if (!isDuration) { + isTimestamp = TInstant::TryParseIso8601(e->Tail().Content(), stamp); + } + } + const TMaybe deadline = options.Config()->ExpirationDeadline.Get(cluster); + const TMaybe interval = options.Config()->ExpirationInterval.Get(cluster); + if (deadline || isTimestamp) { + attrs["expiration_time"] = isTimestamp ? stamp.ToStringUpToSeconds() : deadline->ToStringUpToSeconds(); + } + if (interval || isDuration) { + attrs["expiration_timeout"] = isDuration ? duration.MilliSeconds() : interval->MilliSeconds(); + } + if (options.Config()->NightlyCompress.Get(cluster).GetOrElse(false)) { + attrs["force_nightly_compress"] = true; + } + } + +#define HANDLE_OPT(name, attr, conv) \ + auto dst##name = isAnonymous \ + ? options.Config()->Temporary##name.Get(cluster) \ + : options.Config()->Published##name.Get(cluster); \ + if (auto s = NYql::GetSetting(publish.Settings().Ref(), EYtSettingType::name)) { \ + dst##name = OptionFromString(s->Tail().Content()); \ + } \ + if (dst##name && dst##name != options.Config()->Temporary##name.Get(cluster)) { \ + attrs[attr] = conv(*dst##name); \ + } + + HANDLE_OPT(CompressionCodec, "compression_codec", NoOp); + HANDLE_OPT(ErasureCodec, "erasure_codec", ToString); + HANDLE_OPT(ReplicationFactor, "replication_factor", static_cast); + HANDLE_OPT(Media, "media", NoOp); + HANDLE_OPT(PrimaryMedium, "primary_medium", NoOp); +#undef DEFINE_OPT + + if (auto optimizeFor = options.Config()->OptimizeFor.Get(cluster)) { + if (dstRowSpec->GetType()->GetSize()) { + attrs["optimize_for"] = ToString(*optimizeFor); + } + } + + if (auto ua = NYql::GetSetting(publish.Settings().Ref(), EYtSettingType::UserAttrs)) { + const NYT::TNode mapNode = NYT::NodeFromYsonString(ua->Tail().Content()); + const auto& map = mapNode.AsMap(); + for (auto it = map.cbegin(); it != map.cend(); ++it) { + attrs[it->first] = it->second; + } + } + TOFStream ofAttr(destFilePath + ".attr"); ofAttr.Write(NYT::NodeToYsonString(attrs, NYson::EYsonFormat::Pretty)); } @@ -1037,6 +1116,16 @@ class TYtFileGateway : public IYtGateway { return res; } + TFuture DownloadTables(TDownloadTablesOptions&& options) final { + Y_UNUSED(options); + return MakeFuture(); + } + + TFuture UploadTable(TUploadTableOptions&& options) final { + Y_UNUSED(options); + return MakeFuture(); + } + TFullResultTableResult PrepareFullResultTable(TFullResultTableOptions&& options) final { try { TString cluster = options.Cluster(); diff --git a/ydb/library/yql/providers/yt/gateway/mixed/ya.make b/ydb/library/yql/providers/yt/gateway/mixed/ya.make new file mode 100644 index 000000000000..b4cd75e5d566 --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/mixed/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +SRCS( + yql_yt_mixed.cpp +) + +PEERDIR( + ydb/library/yql/utils/log + ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/gateway/native + ydb/library/yql/providers/yt/gateway/lib + ydb/library/yql/providers/yt/common + ydb/library/yql/providers/common/provider + + library/cpp/threading/future + library/cpp/yson/node +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.cpp b/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.cpp new file mode 100644 index 000000000000..f12a22176830 --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.cpp @@ -0,0 +1,323 @@ +#include "yql_yt_mixed.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + + +namespace NYql { +namespace { + +using namespace NThreading; +using namespace NNodes; + +class TMixedYtGateway final: public IYtGateway { +public: + TMixedYtGateway(const TYtNativeServices& services) + : FileServices_(NFile::TYtFileServices::Make(services.FunctionRegistry, {}, services.FileStorage)) + , FileGateway_(CreateYtFileGateway(FileServices_)) + , NativeGateway_(CreateYtNativeGateway(services)) + { + } + + void OpenSession(TOpenSessionOptions&& options) final { + FileGateway_->OpenSession(TOpenSessionOptions(options)); + NativeGateway_->OpenSession(std::move(options)); + } + + TFuture CloseSession(TCloseSessionOptions&& options) final { + return WaitAll( + FileGateway_->CloseSession(TCloseSessionOptions(options)), + NativeGateway_->CloseSession(std::move(options)) + ); + } + + TFuture CleanupSession(TCleanupSessionOptions&& options) final { + return WaitAll( + FileGateway_->CleanupSession(TCleanupSessionOptions(options)), + NativeGateway_->CleanupSession(std::move(options)) + ); + } + + TFuture Finalize(TFinalizeOptions&& options) final { + for (auto& p : FileServices_->GetTablesMapping()) { + try { + NFs::Remove(p.second); + NFs::Remove(p.second + ".attr"); + } catch (...) { + } + } + + return NativeGateway_->Finalize(std::move(options)); + } + + TFuture CanonizePaths(TCanonizePathsOptions&& options) final { + return NativeGateway_->CanonizePaths(std::move(options)); + } + + TFuture GetTableInfo(TGetTableInfoOptions&& options) final { + std::vector nativeNdx; + std::vector fileNdx; + TGetTableInfoOptions nativeOptions(options); + nativeOptions.Tables().clear(); + TGetTableInfoOptions fileOptions(options); + fileOptions.Tables().clear(); + for (size_t i = 0; i < options.Tables().size(); ++i) { + if (options.Tables()[i].Anonymous()) { + fileOptions.Tables().push_back(options.Tables()[i]); + fileNdx.push_back(i); + } else { + nativeOptions.Tables().push_back(options.Tables()[i]); + nativeNdx.push_back(i); + } + } + TTableInfoResult dummyRes; + dummyRes.SetSuccess(); + TFuture nativeResF = nativeOptions.Tables().empty() ? MakeFuture(dummyRes) : NativeGateway_->GetTableInfo(std::move(nativeOptions)); + TFuture fileResF = fileOptions.Tables().empty() ? MakeFuture(dummyRes) : FileGateway_->GetTableInfo(std::move(fileOptions)); + + return WaitAll(nativeResF.IgnoreResult(), fileResF.IgnoreResult()).Apply([fs = FileServices_, ng = NativeGateway_, options = std::move(options), nativeResF, fileResF, nativeNdx = std::move(nativeNdx), fileNdx = std::move(fileNdx)](const TFuture&) { + auto nativeRes = nativeResF.GetValueSync(); + if (!nativeRes.Success()) { + return nativeResF; + } + auto fileRes = fileResF.GetValueSync(); + if (!fileRes.Success()) { + return fileResF; + } + + TTableInfoResult res; + res.SetStatus(nativeRes.Status()); + res.AddIssues(nativeRes.Issues()); + res.AddIssues(fileRes.Issues()); + res.Data.resize(options.Tables().size()); + for (size_t i = 0; i < fileNdx.size(); ++i) { + res.Data.at(fileNdx[i]) = fileRes.Data.at(i); + } + TVector downloads; + TVector> locks; + TVector> attrs; + for (size_t i = 0; i < nativeNdx.size(); ++i) { + res.Data.at(nativeNdx[i]) = nativeRes.Data.at(i); + const TTableReq& req = options.Tables().at(nativeNdx[i]); + const auto fullTableName = TString(YtProviderName).append('.').append(req.Cluster()).append('.').append(req.Table()); + const auto tablePath = (TFsPath(fs->GetTmpDir()) / CreateGuidAsString()).GetPath(); + + fs->GetTablesMapping()[fullTableName] = tablePath; + if (nativeRes.Data[i].Meta->DoesExist && nativeRes.Data[i].Meta->SqlView.empty()) { + downloads.push_back( + TDownloadTablesReq() + .Cluster(req.Cluster()) + .Table(req.Table()) + .Anonymous(req.Anonymous()) + .TargetPath(tablePath) + ); + locks.emplace_back(tablePath, fullTableName); + NYT::TNode attrsContent = NYT::TNode::CreateMap(); + for (const auto& p: nativeRes.Data[i].Meta->Attrs) { + if (READ_SCHEMA_ATTR_NAME == p.first || YqlRowSpecAttribute == p.first || SCHEMA_ATTR_NAME == p.first || FORMAT_ATTR_NAME == p.first) { + attrsContent[p.first] = NYT::NodeFromYsonString(p.second); + } else if (INFER_SCHEMA_ATTR_NAME == p.first) { + attrsContent["infer_schema"] = true; + } else { + attrsContent[p.first] = p.second; + } + } + attrs.emplace_back(tablePath + ".attr", NYT::NodeToYsonString(attrsContent, ::NYson::EYsonFormat::Pretty)); + } + } + if (!downloads.empty()) { + auto downloadFuture = ng->DownloadTables(TDownloadTablesOptions(options.SessionId()) + .Tables(std::move(downloads)) + .Config(options.Config()) + .Epoch(options.Epoch())); + return downloadFuture.Apply([res = std::move(res), locks = std::move(locks), attrs = std::move(attrs), fs](const TFuture& f) mutable { + try { + auto downloadRes = f.GetValueSync(); + if (!downloadRes.Success()) { + res.SetStatus(downloadRes.Status()); + res.AddIssues(downloadRes.Issues()); + } else { + for (auto& p: locks) { + fs->LockPath(p.first, p.second); + } + for (auto& p: attrs) { + TOFStream ofAttr(p.first); + ofAttr.Write(p.second); + } + } + return res; + } catch (...) { + return ResultFromCurrentException(); + } + }); + } + return MakeFuture(std::move(res)); + }); + } + + TFuture GetTableRange(TTableRangeOptions&& options) final { + return NativeGateway_->GetTableRange(std::move(options)); + } + + TFuture GetFolder(TFolderOptions&& options) final { + return NativeGateway_->GetFolder(std::move(options)); + } + + TFuture ResolveLinks(TResolveOptions&& options) final { + return NativeGateway_->ResolveLinks(std::move(options)); + } + + TFuture GetFolders(TBatchFolderOptions&& options) final { + return NativeGateway_->GetFolders(std::move(options)); + } + + TFuture ResOrPull(const TExprNode::TPtr& node, TExprContext& ctx, TResOrPullOptions&& options) final { + return FileGateway_->ResOrPull(node, ctx, std::move(options)); + } + + TFuture Run(const TExprNode::TPtr& node, TExprContext& ctx, TRunOptions&& options) final { + return FileGateway_->Run(node, ctx, std::move(options)); + } + + TFuture Prepare(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) const final { + return FileGateway_->Prepare(node, ctx, std::move(options)); + } + + TFuture Calc(const TExprNode::TListType& nodes, TExprContext& ctx, TCalcOptions&& options) final { + return FileGateway_->Calc(nodes, ctx, std::move(options)); + } + + TFuture Publish(const TExprNode::TPtr& node, TExprContext& ctx, TPublishOptions&& options) final { + TPublishOptions optionsCopy(options); + auto publish = TYtPublish(node); + auto cluster = publish.DataSink().Cluster().StringValue(); + auto table = publish.Publish().Name().StringValue(); + const bool isAnonymous = NYql::HasSetting(publish.Publish().Settings().Ref(), EYtSettingType::Anonymous); + + auto f = FileGateway_->Publish(node, ctx, std::move(options)); + if (!isAnonymous) { + f = f.Apply([options = std::move(optionsCopy), cluster, table, fs = FileServices_, ng = NativeGateway_](const TFuture& f) { + auto res = f.GetValueSync(); + if (!res.Success()) { + return f; + } + + auto path = fs->GetTablePath(cluster, table, false, true); + TString attrs = "{}"; + if (NFs::Exists(path + ".attr")) { + TIFStream input(path + ".attr"); + attrs = input.ReadAll(); + } + return ng->UploadTable( + TUploadTableOptions(options.SessionId()) + .Cluster(cluster) + .Table(table) + .Path(path) + .Attrs(attrs) + .Config(options.Config()) + ).Apply([res = std::move(res)](const TFuture& f) mutable { + auto uploadRes = f.GetValueSync(); + if (!uploadRes.Success()) { + res.SetStatus(uploadRes.Status()); + res.AddIssues(uploadRes.Issues()); + } + return res; + }); + }); + } + + return f; + } + + TFuture Commit(TCommitOptions&& options) final { + return FileGateway_->Commit(std::move(options)); + } + + TFuture DropTrackables(TDropTrackablesOptions&& options) final { + return FileGateway_->DropTrackables(std::move(options)); + } + + TFuture PathStat(TPathStatOptions&& options) final { + return FileGateway_->PathStat(std::move(options)); + } + + TPathStatResult TryPathStat(TPathStatOptions&& options) final { + return FileGateway_->TryPathStat(std::move(options)); + } + + bool TryParseYtUrl(const TString& url, TString* cluster, TString* path) const final { + return NativeGateway_->TryParseYtUrl(url, cluster, path); + } + + TString GetDefaultClusterName() const final { + return NativeGateway_->GetDefaultClusterName(); + } + + TString GetClusterServer(const TString& cluster) const final { + return NativeGateway_->GetClusterServer(cluster); + } + + NYT::TRichYPath GetRealTable(const TString& sessionId, const TString& cluster, const TString& table, ui32 epoch, const TString& tmpFolder) const final { + return NativeGateway_->GetRealTable(sessionId, cluster, table, epoch, tmpFolder); + } + + NYT::TRichYPath GetWriteTable(const TString& sessionId, const TString& cluster, const TString& table, const TString& tmpFolder) const final { + return NativeGateway_->GetWriteTable(sessionId, cluster, table, tmpFolder); + } + + TFuture DownloadTables(TDownloadTablesOptions&& options) final { + return NativeGateway_->DownloadTables(std::move(options)); + } + + TFuture UploadTable(TUploadTableOptions&& options) final { + return NativeGateway_->UploadTable(std::move(options)); + } + + TFuture GetTableStat(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) final { + return FileGateway_->GetTableStat(node, ctx, std::move(options)); + } + + TFullResultTableResult PrepareFullResultTable(TFullResultTableOptions&& options) final { + return NativeGateway_->PrepareFullResultTable(std::move(options)); + } + + void SetStatUploader(IStatUploader::TPtr statUploader) final { + NativeGateway_->SetStatUploader(statUploader); + } + + void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) final { + FileGateway_->RegisterMkqlCompiler(compiler); + } + + TGetTablePartitionsResult GetTablePartitions(TGetTablePartitionsOptions&& options) override { + return NativeGateway_->GetTablePartitions(std::move(options)); + } + + void AddCluster(const TYtClusterConfig& cluster) override { + NativeGateway_->AddCluster(cluster); + } + +private: + NFile::TYtFileServices::TPtr FileServices_; + IYtGateway::TPtr FileGateway_; + IYtGateway::TPtr NativeGateway_; +}; + +} // namespace + +IYtGateway::TPtr CreateYtMixedGateway(const TYtNativeServices& services) { + return MakeIntrusive(services); +} + +} // namspace NYql diff --git a/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.h b/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.h new file mode 100644 index 000000000000..ed2ca29a296b --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/mixed/yql_yt_mixed.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace NYql { + +IYtGateway::TPtr CreateYtMixedGateway(const TYtNativeServices& services); + +} // namspace NYql diff --git a/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp b/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp index 137134aee03b..59280e6a6eba 100644 --- a/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -1082,6 +1082,133 @@ class TYtNativeGateway : public IYtGateway { return NYT::TRichYPath(table); } + TFuture DownloadTables(TDownloadTablesOptions&& options) final { + YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__); + try { + TSession::TPtr session = GetSession(options.SessionId()); + auto logCtx = NYql::NLog::CurrentLogContextPath(); + + const auto epoch = options.Epoch(); + THashMap entries; + TVector> waits; + for (auto& req: options.Tables()) { + const auto cluster = req.Cluster(); + const auto table = req.Table(); + const auto anon = req.Anonymous(); + const auto targetPath = req.TargetPath(); + YQL_CLOG(DEBUG, ProviderYt) << "Downloading " << cluster << '.' << table << " to " << targetPath; + + TTransactionCache::TEntry::TPtr& entry = entries[cluster]; + if (!entry) { + auto ytServer = Clusters_->TryGetServer(cluster); + YQL_ENSURE(ytServer); + entry = session->TxCache_.GetEntry(ytServer); + } + + auto richYPath = NYT::TRichYPath(table); + if (auto p = entry->Snapshots.FindPtr(std::make_pair(table, epoch))) { + richYPath.Path(std::get<0>(*p)).TransactionId(std::get<1>(*p)); + } else { + auto realTableName = NYql::TransformPath(GetTablesTmpFolder(*options.Config()), table, anon, session->UserName_); + realTableName = NYT::AddPathPrefix(realTableName, NYT::TConfig::Get()->Prefix); + richYPath = NYT::TRichYPath(realTableName); + richYPath.TransactionId(entry->Tx->GetId()); + } + + waits.push_back(session->Queue_->Async([entry, richYPath, targetPath, logCtx] () { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(logCtx); + + auto reader = entry->Tx->CreateRawReader( + richYPath, + NYT::TFormat::YsonText(), + NYT::TTableReaderOptions() + .CreateTransaction(false) + .ControlAttributes( + NYT::TControlAttributes() + .EnableRowIndex(false) + .EnableRangeIndex(false) + ) + ); + + TOFStream out(targetPath); + TransferData(reader.Get(), &out); + out.Finish(); + })); + } + return WaitExceptionOrAll(waits).Apply([](const TFuture& f) { + try { + f.TryRethrow(); + TDownloadTablesResult res; + res.SetSuccess(); + return res; + } catch (...) { + YQL_CLOG(ERROR, ProviderYt) << CurrentExceptionMessage(); + return ResultFromCurrentException(); + } + }); + } catch (...) { + YQL_CLOG(ERROR, ProviderYt) << CurrentExceptionMessage(); + return MakeFuture(ResultFromCurrentException()); + } + } + + TFuture UploadTable(TUploadTableOptions&& options) final { + YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__); + try { + TSession::TPtr session = GetSession(options.SessionId()); + auto logCtx = NYql::NLog::CurrentLogContextPath(); + + const auto cluster = options.Cluster(); + const auto table = options.Table(); + const auto path = options.Path(); + auto attrs = NYT::NodeFromYsonString(options.Attrs()); + const auto config = options.Config(); + YQL_CLOG(DEBUG, ProviderYt) << "Uploading " << path << " to " << cluster << '.' << table; + + auto ytServer = Clusters_->TryGetServer(cluster); + YQL_ENSURE(ytServer); + auto entry = session->TxCache_.GetEntry(ytServer); + + NYT::TTableWriterOptions writerOptions; + auto maxRowWeight = config->MaxRowWeight.Get(cluster); + auto maxKeyWeight = config->MaxKeyWeight.Get(cluster); + + if (maxRowWeight || maxKeyWeight) { + NYT::TNode writeConfig; + if (maxRowWeight) { + writeConfig["max_row_weight"] = static_cast(*maxRowWeight); + } + if (maxKeyWeight) { + writeConfig["max_key_weight"] = static_cast(*maxKeyWeight); + } + writerOptions.Config(writeConfig); + } + + NYT::MergeNodes(attrs, YqlOpOptionsToAttrs(session->OperationOptions_)); + + return session->Queue_->Async([entry, table, path, attrs, writerOptions, logCtx] () { + YQL_LOG_CTX_ROOT_SESSION_SCOPE(logCtx); + try { + entry->Tx->Create(table, NT_TABLE, NYT::TCreateOptions().Force(true).Attributes(attrs)); + + TRawTableWriterPtr writer = entry->Tx->CreateRawWriter(table, NYT::TFormat::YsonText(), writerOptions); + TIFStream in(path); + TransferData(&in, writer.Get()); + writer->Finish(); + TUploadTableResult res; + res.SetSuccess(); + return res; + } catch (...) { + YQL_CLOG(ERROR, ProviderYt) << CurrentExceptionMessage(); + return ResultFromCurrentException(); + } + }); + } catch (...) { + YQL_CLOG(ERROR, ProviderYt) << CurrentExceptionMessage(); + return MakeFuture(ResultFromCurrentException()); + } + } + TFuture Prepare(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) const final { YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__); auto nodePos = ctx.GetPosition(node->Pos()); diff --git a/ydb/library/yql/providers/yt/gateway/profile/yql_yt_profiling.cpp b/ydb/library/yql/providers/yt/gateway/profile/yql_yt_profiling.cpp index ff1b1b51d8e9..4e6fa3cc71d8 100644 --- a/ydb/library/yql/providers/yt/gateway/profile/yql_yt_profiling.cpp +++ b/ydb/library/yql/providers/yt/gateway/profile/yql_yt_profiling.cpp @@ -148,6 +148,18 @@ class TProfilingYtGateway final: public IYtGateway { return Slave_->GetWriteTable(sessionId, cluster, table, tmpFolder); } + TFuture DownloadTables(TDownloadTablesOptions&& options) final { + auto profileScope = YQL_PROFILE_FUNC_VAL(TRACE); + auto future = Slave_->DownloadTables(std::move(options)); + return YQL_PROFILE_BIND_VAL(future, profileScope); + } + + TFuture UploadTable(TUploadTableOptions&& options) final { + auto profileScope = YQL_PROFILE_FUNC_VAL(TRACE); + auto future = Slave_->UploadTable(std::move(options)); + return YQL_PROFILE_BIND_VAL(future, profileScope); + } + NThreading::TFuture GetTableStat(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) final { return Slave_->GetTableStat(node, ctx, std::move(options)); } diff --git a/ydb/library/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp b/ydb/library/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp index c5f878953376..0bd525b28566 100644 --- a/ydb/library/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp +++ b/ydb/library/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp @@ -468,7 +468,7 @@ class TGateway : public IYtGateway { ("Items", itemsNode); return MakeHash(NYT::NodeToCanonicalYsonString(keyNode, NYT::NYson::EYsonFormat::Binary)); - } + } template static NYT::TNode SerializeFolderItem(const T& item) { @@ -712,7 +712,7 @@ class TGateway : public IYtGateway { ("IsTemp", req.IsTemp()) ("IsAnonymous", req.IsAnonymous()) ("Epoch", req.Epoch()); - + node("Path", path); return MakeHash(NYT::NodeToCanonicalYsonString(node)); } @@ -887,6 +887,22 @@ class TGateway : public IYtGateway { return Inner_->GetWriteTable(sessionId, cluster, table, tmpFolder); } + NThreading::TFuture DownloadTables(TDownloadTablesOptions&& options) final { + if (QContext_.CanRead()) { + TDownloadTablesResult res; + res.SetSuccess(); + return NThreading::MakeFuture(std::move(res)); + } + return Inner_->DownloadTables(std::move(options)); + } + + NThreading::TFuture UploadTable(TUploadTableOptions&& options) final { + if (QContext_.CanRead()) { + throw yexception() << "Can't replay UploadTable"; + } + return Inner_->UploadTable(std::move(options)); + } + TFullResultTableResult PrepareFullResultTable(TFullResultTableOptions&& options) final { if (QContext_.CanRead()) { throw yexception() << "Can't replay PrepareFullResultTable"; diff --git a/ydb/library/yql/providers/yt/gateway/ya.make b/ydb/library/yql/providers/yt/gateway/ya.make index 18a3da3ba3bf..404e4230146f 100644 --- a/ydb/library/yql/providers/yt/gateway/ya.make +++ b/ydb/library/yql/providers/yt/gateway/ya.make @@ -1,6 +1,7 @@ RECURSE( file lib + mixed native profile qplayer diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_gateway.h b/ydb/library/yql/providers/yt/provider/yql_yt_gateway.h index 98f0ba50f0af..55bb3b4d76f9 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_gateway.h +++ b/ydb/library/yql/providers/yt/provider/yql_yt_gateway.h @@ -570,6 +570,49 @@ class IYtGateway : public TThrRefBase { NYT::TMultiTablePartitions Partitions; }; + struct TDownloadTablesReq { + using TSelf = TDownloadTablesReq; + + OPTION_FIELD(TString, Cluster) + OPTION_FIELD(TString, Table) + OPTION_FIELD_DEFAULT(bool, Anonymous, false) + OPTION_FIELD(TString, TargetPath) + }; + + struct TDownloadTablesOptions : public TCommonOptions { + using TSelf = TDownloadTablesOptions; + + TDownloadTablesOptions(const TString& sessionId) + : TCommonOptions(sessionId) + { + } + + OPTION_FIELD(TVector, Tables) + OPTION_FIELD_DEFAULT(ui32, Epoch, 0) + OPTION_FIELD(TYtSettings::TConstPtr, Config) + }; + + struct TDownloadTablesResult: public NCommon::TOperationResult { + }; + + struct TUploadTableOptions : public TCommonOptions { + using TSelf = TUploadTableOptions; + + TUploadTableOptions(const TString& sessionId) + : TCommonOptions(sessionId) + { + } + + OPTION_FIELD(TString, Cluster) + OPTION_FIELD(TString, Table) + OPTION_FIELD(TString, Path) + OPTION_FIELD(TString, Attrs) + OPTION_FIELD(TYtSettings::TConstPtr, Config) + }; + + struct TUploadTableResult: public NCommon::TOperationResult { + }; + public: virtual ~IYtGateway() = default; @@ -618,6 +661,9 @@ class IYtGateway : public TThrRefBase { virtual NYT::TRichYPath GetRealTable(const TString& sessionId, const TString& cluster, const TString& table, ui32 epoch, const TString& tmpFolder) const = 0; virtual NYT::TRichYPath GetWriteTable(const TString& sessionId, const TString& cluster, const TString& table, const TString& tmpFolder) const = 0; + virtual NThreading::TFuture DownloadTables(TDownloadTablesOptions&& options) = 0; + virtual NThreading::TFuture UploadTable(TUploadTableOptions&& options) = 0; + virtual TFullResultTableResult PrepareFullResultTable(TFullResultTableOptions&& options) = 0; virtual void SetStatUploader(IStatUploader::TPtr statUploader) = 0; diff --git a/ydb/library/yql/tests/s-expressions/yt_native_file/part9/canondata/result.json b/ydb/library/yql/tests/s-expressions/yt_native_file/part9/canondata/result.json index 7069c37ba3e6..433f1072273a 100644 --- a/ydb/library/yql/tests/s-expressions/yt_native_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/s-expressions/yt_native_file/part9/canondata/result.json @@ -419,9 +419,9 @@ "uri": "https://{canondata_backend}/1925842/fc5ed2cd723535760526f509cc186349095a298d/resource.tar.gz#test.test_Epochs-AppendToSortedBySorted-Debug_/opt.yql" }, { - "checksum": "8a7e4b22223981603ef46d1e6e4dafdd", + "checksum": "d568066d2f7dc95a0e9fe10d1d006379", "size": 640, - "uri": "https://{canondata_backend}/1031349/f3449c18b7be2569111eb320b6d7808083fdafa0/resource.tar.gz#test.test_Epochs-AppendToSortedBySorted-Debug_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1917492/ff660757c26cf68a3348b7d864b73abaadbef456/resource.tar.gz#test.test_Epochs-AppendToSortedBySorted-Debug_/Output.yqlrun.txt.attr" }, { "checksum": "032545bf1b34e6c72b3705c57e98a17f", @@ -455,9 +455,9 @@ "uri": "https://{canondata_backend}/1925842/fc5ed2cd723535760526f509cc186349095a298d/resource.tar.gz#test.test_Epochs-AppendToSortedBySortedAfterCommit-Debug_/opt.yql" }, { - "checksum": "8a7e4b22223981603ef46d1e6e4dafdd", + "checksum": "d568066d2f7dc95a0e9fe10d1d006379", "size": 640, - "uri": "https://{canondata_backend}/1031349/f3449c18b7be2569111eb320b6d7808083fdafa0/resource.tar.gz#test.test_Epochs-AppendToSortedBySortedAfterCommit-Debug_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1917492/ff660757c26cf68a3348b7d864b73abaadbef456/resource.tar.gz#test.test_Epochs-AppendToSortedBySortedAfterCommit-Debug_/Output.yqlrun.txt.attr" }, { "checksum": "032545bf1b34e6c72b3705c57e98a17f", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json index 0758cea9cf2f..01724867b165 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json @@ -929,9 +929,9 @@ "uri": "https://{canondata_backend}/1880306/fe7813445a8795b19ff79ce4c90ea7375ca0d21c/resource.tar.gz#test.test_column_group-publish-single-Results_/Output.txt" }, { - "checksum": "eb2d8b435e428e956f1bafb75ba6af84", - "size": 916, - "uri": "https://{canondata_backend}/1880306/fe7813445a8795b19ff79ce4c90ea7375ca0d21c/resource.tar.gz#test.test_column_group-publish-single-Results_/Output.yqlrun.txt.attr" + "checksum": "98d916f86f56d14e04ab1219419c27f3", + "size": 945, + "uri": "https://{canondata_backend}/1937367/48d786158af84bcb30baabe19ed069b1ef5b48fb/resource.tar.gz#test.test_column_group-publish-single-Results_/Output.yqlrun.txt.attr" } ], "test.test[column_order-insert_reorder_without_columnorder--Debug]": [ @@ -1397,9 +1397,9 @@ "uri": "https://{canondata_backend}/1871102/79f07c08940ddd8c31e112c30d9b724aec21c4b4/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output1.txt" }, { - "checksum": "8a7e4b22223981603ef46d1e6e4dafdd", + "checksum": "d568066d2f7dc95a0e9fe10d1d006379", "size": 640, - "uri": "https://{canondata_backend}/1817427/6caf82e8e48e0e3583742d406be945918c7f60a0/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output1.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/8a58697a64a1552f1ef148251bf40b960f549bdd/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output1.yqlrun.txt.attr" }, { "checksum": "10c1323e063a4f213111e200ed510a0b", @@ -1407,9 +1407,9 @@ "uri": "https://{canondata_backend}/1847551/a8d4b58a85ca0756ae0eaefba02ac6937ce6bd41/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output2.txt" }, { - "checksum": "8a7e4b22223981603ef46d1e6e4dafdd", + "checksum": "d568066d2f7dc95a0e9fe10d1d006379", "size": 640, - "uri": "https://{canondata_backend}/1817427/6caf82e8e48e0e3583742d406be945918c7f60a0/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output2.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/8a58697a64a1552f1ef148251bf40b960f549bdd/resource.tar.gz#test.test_insert-yql-13083-existig-Results_/Output2.yqlrun.txt.attr" }, { "uri": "file://test.test_insert-yql-13083-existig-Results_/extracted" @@ -1436,9 +1436,9 @@ "uri": "https://{canondata_backend}/1937150/005acb6ac365531cce4336a5ae255c46c4767b16/resource.tar.gz#test.test_insert_monotonic-from_empty--Results_/Output.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1817427/6caf82e8e48e0e3583742d406be945918c7f60a0/resource.tar.gz#test.test_insert_monotonic-from_empty--Results_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/8a58697a64a1552f1ef148251bf40b960f549bdd/resource.tar.gz#test.test_insert_monotonic-from_empty--Results_/Output.yqlrun.txt.attr" } ], "test.test[join-convert_key--Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json index 3f0b5516a241..b70bc924ca02 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json @@ -742,9 +742,9 @@ "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output4.txt" }, { - "checksum": "eb2d8b435e428e956f1bafb75ba6af84", - "size": 916, - "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output4.yqlrun.txt.attr" + "checksum": "98d916f86f56d14e04ab1219419c27f3", + "size": 945, + "uri": "https://{canondata_backend}/1923547/079ed525adfefb739e07ac45dcf5e817b56cc8a3/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output4.yqlrun.txt.attr" }, { "checksum": "94fde3f913511b2af79a70f97849f2c3", @@ -752,9 +752,9 @@ "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output1.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output1.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1923547/079ed525adfefb739e07ac45dcf5e817b56cc8a3/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output1.yqlrun.txt.attr" }, { "checksum": "51d987b4ca321d62e67eab49cc8d6622", @@ -762,9 +762,9 @@ "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output2.txt" }, { - "checksum": "caae862df12a053afc8ba3d160f1ea4a", - "size": 1090, - "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output2.yqlrun.txt.attr" + "checksum": "d1d5b0d63b6432f84d60d73ee01480f2", + "size": 1119, + "uri": "https://{canondata_backend}/1923547/079ed525adfefb739e07ac45dcf5e817b56cc8a3/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output2.yqlrun.txt.attr" }, { "checksum": "a0ea00435e4da70c27100575ce1b1708", @@ -772,9 +772,9 @@ "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output3.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output3.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1923547/079ed525adfefb739e07ac45dcf5e817b56cc8a3/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output3.yqlrun.txt.attr" } ], "test.test[column_group-many_inserts--Debug]": [ @@ -798,9 +798,9 @@ "uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Results_/Output.txt" }, { - "checksum": "685987a135166ca30afe0a22810c35c3", - "size": 1078, - "uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Results_/Output.yqlrun.txt.attr" + "checksum": "42cd02950a5b1d0c1ec5e61755e52913", + "size": 1107, + "uri": "https://{canondata_backend}/1923547/079ed525adfefb739e07ac45dcf5e817b56cc8a3/resource.tar.gz#test.test_column_group-many_inserts--Results_/Output.yqlrun.txt.attr" } ], "test.test[column_order-winfunc-default.txt-Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json index 243263180d3c..897ce3902b02 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json @@ -1144,9 +1144,9 @@ "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output1.txt" }, { - "checksum": "8a7e4b22223981603ef46d1e6e4dafdd", + "checksum": "d568066d2f7dc95a0e9fe10d1d006379", "size": 640, - "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output1.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/0d01350a356eff5d429590af88c3c870595fbf83/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output1.yqlrun.txt.attr" }, { "checksum": "ea3d4691fe1afcac13d779d3badc0f39", @@ -1154,9 +1154,9 @@ "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output2.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output2.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/0d01350a356eff5d429590af88c3c870595fbf83/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output2.yqlrun.txt.attr" }, { "checksum": "ea3d4691fe1afcac13d779d3badc0f39", @@ -1164,9 +1164,9 @@ "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output3.txt" }, { - "checksum": "88f5c2e1cc8d6ea022c9376acb9985eb", - "size": 720, - "uri": "https://{canondata_backend}/1936842/f59767c74584bc86205ef6d438da50749ba49f6c/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output3.yqlrun.txt.attr" + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", + "size": 680, + "uri": "https://{canondata_backend}/1937492/0d01350a356eff5d429590af88c3c870595fbf83/resource.tar.gz#test.test_insert-append_sorted-to_sorted-Results_/Output3.yqlrun.txt.attr" }, { "uri": "file://test.test_insert-append_sorted-to_sorted-Results_/extracted" diff --git a/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json index 07ebe3af3513..d9f6a4f74acb 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json @@ -673,9 +673,9 @@ "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output4.txt" }, { - "checksum": "eb2d8b435e428e956f1bafb75ba6af84", - "size": 916, - "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output4.yqlrun.txt.attr" + "checksum": "98d916f86f56d14e04ab1219419c27f3", + "size": 945, + "uri": "https://{canondata_backend}/1937492/6431b9a9ee0c0a4606f9d3fd10fca9a9ab0fcb81/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output4.yqlrun.txt.attr" }, { "checksum": "94fde3f913511b2af79a70f97849f2c3", @@ -683,9 +683,9 @@ "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output1.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output1.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1937492/6431b9a9ee0c0a4606f9d3fd10fca9a9ab0fcb81/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output1.yqlrun.txt.attr" }, { "checksum": "51d987b4ca321d62e67eab49cc8d6622", @@ -693,9 +693,9 @@ "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output2.txt" }, { - "checksum": "caae862df12a053afc8ba3d160f1ea4a", - "size": 1090, - "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output2.yqlrun.txt.attr" + "checksum": "d1d5b0d63b6432f84d60d73ee01480f2", + "size": 1119, + "uri": "https://{canondata_backend}/1937492/6431b9a9ee0c0a4606f9d3fd10fca9a9ab0fcb81/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output2.yqlrun.txt.attr" }, { "checksum": "a0ea00435e4da70c27100575ce1b1708", @@ -703,9 +703,9 @@ "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output3.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/212715/4061de6555ccf29cd4ed5b782da6de3bcefa3a67/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output3.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1937492/6431b9a9ee0c0a4606f9d3fd10fca9a9ab0fcb81/resource.tar.gz#test.test_column_group-hint-perusage-Results_/Output3.yqlrun.txt.attr" } ], "test.test[column_group-hint_unk_col_fail--Debug]": [], @@ -1171,9 +1171,9 @@ "uri": "https://{canondata_backend}/1924537/d66bfe69aa802f6a81aadbac897621b721f31b4b/resource.tar.gz#test.test_insert_monotonic-to_empty--Results_/EmptyOutput.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1936997/92e29bcf390facb4c009125d036b189dd5abf261/resource.tar.gz#test.test_insert_monotonic-to_empty--Results_/EmptyOutput.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1916746/f63ad7b34e9dba440fab47205b0bd80497c7c687/resource.tar.gz#test.test_insert_monotonic-to_empty--Results_/EmptyOutput.yqlrun.txt.attr" } ], "test.test[join-bush_in--Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json index 286a00cd359a..5615da979804 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json @@ -651,9 +651,9 @@ "uri": "https://{canondata_backend}/1689644/095e0719ce2f673ac6ebc930715493c7d153eec2/resource.tar.gz#test.test_column_group-publish-perusage-Results_/Output.txt" }, { - "checksum": "eb2d8b435e428e956f1bafb75ba6af84", - "size": 916, - "uri": "https://{canondata_backend}/1689644/095e0719ce2f673ac6ebc930715493c7d153eec2/resource.tar.gz#test.test_column_group-publish-perusage-Results_/Output.yqlrun.txt.attr" + "checksum": "98d916f86f56d14e04ab1219419c27f3", + "size": 945, + "uri": "https://{canondata_backend}/1942100/c2ad94f7a5b439844e262d6b8c193c1141b41b70/resource.tar.gz#test.test_column_group-publish-perusage-Results_/Output.yqlrun.txt.attr" } ], "test.test[column_order-align_publish_native--Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json index c43b61433382..886aedd20118 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json @@ -1194,9 +1194,9 @@ "uri": "https://{canondata_backend}/1942173/9fdef87b229faa39ff0555e93cba84e1c922d701/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output6.txt" }, { - "checksum": "d90a999c4560af70083981533c9dd4bc", + "checksum": "e828f6064851f78707c7cd0626287a30", "size": 1297, - "uri": "https://{canondata_backend}/1925842/afffefe6f28e755f103208e94a21f57484ab9ec5/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output6.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1942415/1e39e9497348650f53c9b3b1e91fe758f9c2b2a3/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output6.yqlrun.txt.attr" }, { "checksum": "9f96075b33b117bdb63abd84ee259638", @@ -1204,9 +1204,9 @@ "uri": "https://{canondata_backend}/1942173/9fdef87b229faa39ff0555e93cba84e1c922d701/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output7.txt" }, { - "checksum": "30904ba93f19bf466cbdc861175176bc", - "size": 1337, - "uri": "https://{canondata_backend}/1925842/afffefe6f28e755f103208e94a21f57484ab9ec5/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output7.yqlrun.txt.attr" + "checksum": "e828f6064851f78707c7cd0626287a30", + "size": 1297, + "uri": "https://{canondata_backend}/1942415/1e39e9497348650f53c9b3b1e91fe758f9c2b2a3/resource.tar.gz#test.test_insert-append_sorted-to_sorted_desc-Results_/Output7.yqlrun.txt.attr" }, { "checksum": "581a11fa0d18ebf4212062107a0a2f3f", @@ -1289,9 +1289,9 @@ "uri": "https://{canondata_backend}/1937027/67f47e98bc150e7387e3e30adedc957caafe9c32/resource.tar.gz#test.test_insert_monotonic-several1-default.txt-Results_/Output.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1925842/afffefe6f28e755f103208e94a21f57484ab9ec5/resource.tar.gz#test.test_insert_monotonic-several1-default.txt-Results_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1942415/1e39e9497348650f53c9b3b1e91fe758f9c2b2a3/resource.tar.gz#test.test_insert_monotonic-several1-default.txt-Results_/Output.yqlrun.txt.attr" } ], "test.test[join-force_merge_join-default.txt-Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part19/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part19/canondata/result.json index 833b8c559e4e..b0cb15b23b93 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part19/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part19/canondata/result.json @@ -1259,9 +1259,9 @@ "uri": "https://{canondata_backend}/1937027/16b7289b1b8f5fdff728155d836fa2b238949b2d/resource.tar.gz#test.test_insert_monotonic-truncate_and_append-default.txt-Results_/Output.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1775319/c18d3af89fd62644ccc5174aaafa5d9e5a9091ef/resource.tar.gz#test.test_insert_monotonic-truncate_and_append-default.txt-Results_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1937492/8539c221d3ddc366f32d5aa23ff2d26e142b8238/resource.tar.gz#test.test_insert_monotonic-truncate_and_append-default.txt-Results_/Output.yqlrun.txt.attr" } ], "test.test[join-equi_join_by_expr--Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json index 618d283c81b3..c98713374d34 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json @@ -931,9 +931,9 @@ "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output4.txt" }, { - "checksum": "eb2d8b435e428e956f1bafb75ba6af84", - "size": 916, - "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output4.yqlrun.txt.attr" + "checksum": "98d916f86f56d14e04ab1219419c27f3", + "size": 945, + "uri": "https://{canondata_backend}/1923547/e603142eb3197fe3994d0f9592d2244d0aeff37e/resource.tar.gz#test.test_column_group-hint-single-Results_/Output4.yqlrun.txt.attr" }, { "checksum": "94fde3f913511b2af79a70f97849f2c3", @@ -941,9 +941,9 @@ "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output1.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output1.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1923547/e603142eb3197fe3994d0f9592d2244d0aeff37e/resource.tar.gz#test.test_column_group-hint-single-Results_/Output1.yqlrun.txt.attr" }, { "checksum": "51d987b4ca321d62e67eab49cc8d6622", @@ -951,9 +951,9 @@ "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output2.txt" }, { - "checksum": "caae862df12a053afc8ba3d160f1ea4a", - "size": 1090, - "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output2.yqlrun.txt.attr" + "checksum": "d1d5b0d63b6432f84d60d73ee01480f2", + "size": 1119, + "uri": "https://{canondata_backend}/1923547/e603142eb3197fe3994d0f9592d2244d0aeff37e/resource.tar.gz#test.test_column_group-hint-single-Results_/Output2.yqlrun.txt.attr" }, { "checksum": "a0ea00435e4da70c27100575ce1b1708", @@ -961,9 +961,9 @@ "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output3.txt" }, { - "checksum": "7944c7beb28ff0a6907041f5b0a2af1b", - "size": 1087, - "uri": "https://{canondata_backend}/1809005/968c44f48b1a83965c869c6c3218e862c519db88/resource.tar.gz#test.test_column_group-hint-single-Results_/Output3.yqlrun.txt.attr" + "checksum": "5d11a5046b0adaf24dbb4cdc4f8f3c30", + "size": 1116, + "uri": "https://{canondata_backend}/1923547/e603142eb3197fe3994d0f9592d2244d0aeff37e/resource.tar.gz#test.test_column_group-hint-single-Results_/Output3.yqlrun.txt.attr" } ], "test.test[column_group-hint_anon_groups-perusage-Debug]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json index 95394cccdccb..89d7455bf448 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json @@ -1228,9 +1228,9 @@ "uri": "https://{canondata_backend}/1937150/fdc25242203d3ca2744aaee5d18ad2fe54bedae8/resource.tar.gz#test.test_insert_monotonic-several2-default.txt-Results_/Output.txt" }, { - "checksum": "7262952f81e108293bd219955828acaf", + "checksum": "60f25d4e572a0d9f87a014fdb6e5b921", "size": 680, - "uri": "https://{canondata_backend}/1942415/7dbeab20f9e45cf16ec6f586b5def65a0d68405a/resource.tar.gz#test.test_insert_monotonic-several2-default.txt-Results_/Output.yqlrun.txt.attr" + "uri": "https://{canondata_backend}/1784117/102bef14d3362be8d63290049907f5e996fbefaa/resource.tar.gz#test.test_insert_monotonic-several2-default.txt-Results_/Output.yqlrun.txt.attr" } ], "test.test[join-compact_join--Debug]": [