forked from ydb-platform/ydb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
scanners unification plain/simple for reuse code (ydb-platform#12847)
- Loading branch information
1 parent
c3f10e1
commit d500b87
Showing
28 changed files
with
1,104 additions
and
1,440 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| #include "fetch_steps.h" | ||
| #include "source.h" | ||
|
|
||
| #include <ydb/core/formats/arrow/common/container.h> | ||
| #include <ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h> | ||
| #include <ydb/core/tx/conveyor/usage/service.h> | ||
| #include <ydb/core/tx/limiter/grouped_memory/usage/service.h> | ||
|
|
||
| #include <ydb/library/formats/arrow/simple_arrays_cache.h> | ||
|
|
||
| namespace NKikimr::NOlap::NReader::NCommon { | ||
|
|
||
| TConclusion<bool> TColumnBlobsFetchingStep::DoExecuteInplace( | ||
| const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const { | ||
| return !source->StartFetchingColumns(source, step, Columns); | ||
| } | ||
|
|
||
| ui64 TColumnBlobsFetchingStep::GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const { | ||
| return source->GetColumnBlobBytes(Columns.GetColumnIds()); | ||
| } | ||
|
|
||
| TConclusion<bool> TAssemblerStep::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const { | ||
| source->AssembleColumns(Columns); | ||
| return true; | ||
| } | ||
|
|
||
| ui64 TAssemblerStep::GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const { | ||
| return source->GetColumnRawBytes(Columns->GetColumnIds()); | ||
| } | ||
|
|
||
| TConclusion<bool> TOptionalAssemblerStep::DoExecuteInplace( | ||
| const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const { | ||
| source->AssembleColumns(Columns, !source->IsSourceInMemory()); | ||
| return true; | ||
| } | ||
|
|
||
| ui64 TOptionalAssemblerStep::GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const { | ||
| return source->GetColumnsVolume(Columns->GetColumnIds(), EMemType::RawSequential); | ||
| } | ||
|
|
||
| bool TAllocateMemoryStep::TFetchingStepAllocation::DoOnAllocated(std::shared_ptr<NGroupedMemoryManager::TAllocationGuard>&& guard, | ||
| const std::shared_ptr<NGroupedMemoryManager::IAllocation>& /*allocation*/) { | ||
| auto data = Source.lock(); | ||
| if (!data || data->GetContext()->IsAborted()) { | ||
| guard->Release(); | ||
| return false; | ||
| } | ||
| if (StageIndex == EStageFeaturesIndexes::Accessors) { | ||
| data->MutableStageData().SetAccessorsGuard(std::move(guard)); | ||
| } else { | ||
| data->RegisterAllocationGuard(std::move(guard)); | ||
| } | ||
| Step.Next(); | ||
| auto task = std::make_shared<TStepAction>(data, std::move(Step), data->GetContext()->GetCommonContext()->GetScanActorId()); | ||
| NConveyor::TScanServiceOperator::SendTaskToExecute(task); | ||
| return true; | ||
| } | ||
|
|
||
| TAllocateMemoryStep::TFetchingStepAllocation::TFetchingStepAllocation( | ||
| const std::shared_ptr<IDataSource>& source, const ui64 mem, const TFetchingScriptCursor& step, const EStageFeaturesIndexes stageIndex) | ||
| : TBase(mem) | ||
| , Source(source) | ||
| , Step(step) | ||
| , TasksGuard(source->GetContext()->GetCommonContext()->GetCounters().GetResourcesAllocationTasksGuard()) | ||
| , StageIndex(stageIndex) { | ||
| } | ||
|
|
||
| void TAllocateMemoryStep::TFetchingStepAllocation::DoOnAllocationImpossible(const TString& errorMessage) { | ||
| auto sourcePtr = Source.lock(); | ||
| if (sourcePtr) { | ||
| sourcePtr->GetContext()->GetCommonContext()->AbortWithError( | ||
| "cannot allocate memory for step " + Step.GetName() + ": '" + errorMessage + "'"); | ||
| } | ||
| } | ||
|
|
||
| TConclusion<bool> TAllocateMemoryStep::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const { | ||
| ui64 size = PredefinedSize.value_or(0); | ||
| for (auto&& i : Packs) { | ||
| ui32 sizeLocal = source->GetColumnsVolume(i.GetColumns().GetColumnIds(), i.GetMemType()); | ||
| if (source->GetStageData().GetUseFilter() && i.GetMemType() != EMemType::Blob && source->GetContext()->GetReadMetadata()->HasLimit()) { | ||
| const ui32 filtered = | ||
| source->GetStageData().GetFilteredCount(source->GetRecordsCount(), source->GetContext()->GetReadMetadata()->GetLimitRobust()); | ||
| if (filtered < source->GetRecordsCount()) { | ||
| sizeLocal = sizeLocal * 1.0 * filtered / source->GetRecordsCount(); | ||
| } | ||
| } | ||
| size += sizeLocal; | ||
| } | ||
|
|
||
| auto allocation = std::make_shared<TFetchingStepAllocation>(source, size, step, StageIndex); | ||
| NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation(source->GetContext()->GetProcessMemoryControlId(), | ||
| source->GetContext()->GetCommonContext()->GetScanId(), source->GetMemoryGroupId(), { allocation }, (ui32)StageIndex); | ||
| return false; | ||
| } | ||
|
|
||
| ui64 TAllocateMemoryStep::GetProcessingDataSize(const std::shared_ptr<IDataSource>& /*source*/) const { | ||
| return 0; | ||
| } | ||
|
|
||
| NKikimr::TConclusion<bool> TBuildStageResultStep::DoExecuteInplace( | ||
| const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const { | ||
| source->BuildStageResult(source); | ||
| return true; | ||
| } | ||
|
|
||
| } // namespace NKikimr::NOlap::NReader::NCommon |
146 changes: 146 additions & 0 deletions
146
ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| #pragma once | ||
| #include "fetching.h" | ||
|
|
||
| #include <ydb/core/tx/limiter/grouped_memory/usage/abstract.h> | ||
|
|
||
| namespace NKikimr::NOlap::NReader::NCommon { | ||
|
|
||
| class TAllocateMemoryStep: public IFetchingStep { | ||
| private: | ||
| using TBase = IFetchingStep; | ||
| class TColumnsPack { | ||
| private: | ||
| YDB_READONLY_DEF(TColumnsSetIds, Columns); | ||
| YDB_READONLY(EMemType, MemType, EMemType::Blob); | ||
|
|
||
| public: | ||
| TColumnsPack(const TColumnsSetIds& columns, const EMemType memType) | ||
| : Columns(columns) | ||
| , MemType(memType) { | ||
| } | ||
| }; | ||
| std::vector<TColumnsPack> Packs; | ||
| THashMap<ui32, THashSet<EMemType>> Control; | ||
| const EStageFeaturesIndexes StageIndex; | ||
| const std::optional<ui64> PredefinedSize; | ||
|
|
||
| protected: | ||
| class TFetchingStepAllocation: public NGroupedMemoryManager::IAllocation { | ||
| private: | ||
| using TBase = NGroupedMemoryManager::IAllocation; | ||
| std::weak_ptr<IDataSource> Source; | ||
| TFetchingScriptCursor Step; | ||
| NColumnShard::TCounterGuard TasksGuard; | ||
| const EStageFeaturesIndexes StageIndex; | ||
| virtual bool DoOnAllocated(std::shared_ptr<NGroupedMemoryManager::TAllocationGuard>&& guard, | ||
| const std::shared_ptr<NGroupedMemoryManager::IAllocation>& allocation) override; | ||
| virtual void DoOnAllocationImpossible(const TString& errorMessage) override; | ||
|
|
||
| public: | ||
| TFetchingStepAllocation(const std::shared_ptr<IDataSource>& source, const ui64 mem, const TFetchingScriptCursor& step, | ||
| const EStageFeaturesIndexes stageIndex); | ||
| }; | ||
| virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; | ||
| virtual ui64 GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const override; | ||
| virtual TString DoDebugString() const override { | ||
| return TStringBuilder() << "stage=" << StageIndex << ";"; | ||
| } | ||
|
|
||
| public: | ||
| void AddAllocation(const TColumnsSetIds& ids, const EMemType memType) { | ||
| if (!ids.GetColumnsCount()) { | ||
| return; | ||
| } | ||
| for (auto&& i : ids.GetColumnIds()) { | ||
| AFL_VERIFY(Control[i].emplace(memType).second); | ||
| } | ||
| Packs.emplace_back(ids, memType); | ||
| } | ||
| EStageFeaturesIndexes GetStage() const { | ||
| return StageIndex; | ||
| } | ||
|
|
||
| TAllocateMemoryStep(const TColumnsSetIds& columns, const EMemType memType, const EStageFeaturesIndexes stageIndex) | ||
| : TBase("ALLOCATE_MEMORY::" + ::ToString(stageIndex)) | ||
| , StageIndex(stageIndex) { | ||
| AddAllocation(columns, memType); | ||
| } | ||
|
|
||
| TAllocateMemoryStep(const ui64 memSize, const EStageFeaturesIndexes stageIndex) | ||
| : TBase("ALLOCATE_MEMORY::" + ::ToString(stageIndex)) | ||
| , StageIndex(stageIndex) | ||
| , PredefinedSize(memSize) { | ||
| } | ||
| }; | ||
|
|
||
| class TAssemblerStep: public IFetchingStep { | ||
| private: | ||
| using TBase = IFetchingStep; | ||
| YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, Columns); | ||
| virtual TString DoDebugString() const override { | ||
| return TStringBuilder() << "columns=" << Columns->DebugString() << ";"; | ||
| } | ||
|
|
||
| public: | ||
| virtual ui64 GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const override; | ||
| virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; | ||
| TAssemblerStep(const std::shared_ptr<TColumnsSet>& columns, const TString& specName = Default<TString>()) | ||
| : TBase("ASSEMBLER" + (specName ? "::" + specName : "")) | ||
| , Columns(columns) { | ||
| AFL_VERIFY(Columns); | ||
| AFL_VERIFY(Columns->GetColumnsCount()); | ||
| } | ||
| }; | ||
|
|
||
| class TBuildStageResultStep: public IFetchingStep { | ||
| private: | ||
| using TBase = IFetchingStep; | ||
|
|
||
| public: | ||
| virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const override; | ||
| TBuildStageResultStep() | ||
| : TBase("BUILD_STAGE_RESULT") { | ||
| } | ||
| }; | ||
|
|
||
| class TOptionalAssemblerStep: public IFetchingStep { | ||
| private: | ||
| using TBase = IFetchingStep; | ||
| YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, Columns); | ||
| virtual TString DoDebugString() const override { | ||
| return TStringBuilder() << "columns=" << Columns->DebugString() << ";"; | ||
| } | ||
|
|
||
| public: | ||
| virtual ui64 GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const override; | ||
|
|
||
| virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; | ||
| TOptionalAssemblerStep(const std::shared_ptr<TColumnsSet>& columns, const TString& specName = Default<TString>()) | ||
| : TBase("OPTIONAL_ASSEMBLER" + (specName ? "::" + specName : "")) | ||
| , Columns(columns) { | ||
| AFL_VERIFY(Columns); | ||
| AFL_VERIFY(Columns->GetColumnsCount()); | ||
| } | ||
| }; | ||
|
|
||
| class TColumnBlobsFetchingStep: public IFetchingStep { | ||
| private: | ||
| using TBase = IFetchingStep; | ||
| TColumnsSetIds Columns; | ||
|
|
||
| protected: | ||
| virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; | ||
| virtual TString DoDebugString() const override { | ||
| return TStringBuilder() << "columns=" << Columns.DebugString() << ";"; | ||
| } | ||
|
|
||
| public: | ||
| virtual ui64 GetProcessingDataSize(const std::shared_ptr<IDataSource>& source) const override; | ||
| TColumnBlobsFetchingStep(const TColumnsSetIds& columns) | ||
| : TBase("FETCHING_COLUMNS") | ||
| , Columns(columns) { | ||
| AFL_VERIFY(Columns.GetColumnsCount()); | ||
| } | ||
| }; | ||
|
|
||
| } // namespace NKikimr::NOlap::NReader::NCommon |
Oops, something went wrong.