From 515b64dd6d25a9cb577a7ce3c459464102767109 Mon Sep 17 00:00:00 2001 From: nsofya Date: Thu, 16 May 2024 23:11:17 +0300 Subject: [PATCH] Sync with 24 1 (#4628) Co-authored-by: nsofya --- ydb/core/tx/columnshard/columnshard__init.cpp | 2 ++ .../tx/columnshard/normalizer/abstract/abstract.h | 15 +++++++++------ .../columnshard/normalizer/granule/normalizer.cpp | 4 ++++ .../tx/columnshard/normalizer/portion/chunks.cpp | 4 ++++ .../tx/columnshard/normalizer/portion/clean.cpp | 4 ++++ .../tx/columnshard/normalizer/portion/portion.cpp | 4 ++++ .../columnshard/normalizer/tables/normalizer.cpp | 4 ++++ ydb/core/tx/schemeshard/ut_olap/ya.make | 1 + 8 files changed, 32 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index d7096352a010..82f1b114ec2d 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -272,6 +272,7 @@ bool TTxUpdateSchema::Execute(TTransactionContext& txc, const TActorContext&) { if (result.IsSuccess()) { NormalizerTasks = result.DetachResult(); if (!NormalizerTasks.empty()) { + ACFL_WARN("normalizer_controller", Self->NormalizerController.DebugString())("tasks_count", NormalizerTasks.size()); break; } NIceDb::TNiceDb db(txc.DB); @@ -337,6 +338,7 @@ bool TTxApplyNormalizer::Execute(TTransactionContext& txc, const TActorContext&) void TTxApplyNormalizer::Complete(const TActorContext& ctx) { AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("step", "TTxApplyNormalizer.Complete")("tablet_id", Self->TabletID())("event", "initialize_shard"); AFL_VERIFY(!Self->NormalizerController.IsNormalizationFinished())("details", Self->NormalizerController.DebugString()); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "apply_normalizer_changes")("details", Self->NormalizerController.DebugString())("size", Changes->GetSize()); Changes->ApplyOnComplete(Self->NormalizerController); Self->NormalizerController.GetNormalizer()->OnResultReady(); if (Self->NormalizerController.GetNormalizer()->HasActiveTasks()) { diff --git a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h index 592bc325dbe7..fa107c7fcc05 100644 --- a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h +++ b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h @@ -53,9 +53,9 @@ namespace NKikimr::NOlap { enum class ENormalizerSequentialId : ui32 { Granules = 1, Chunks, - PortionsMetadata, PortionsCleaner, - TablesCleaner + TablesCleaner, + PortionsMetadata, }; class TNormalizationContext { @@ -88,6 +88,8 @@ namespace NKikimr::NOlap { virtual void ApplyOnComplete(const TNormalizationController& normalizationContext) const { Y_UNUSED(normalizationContext); } + + virtual ui64 GetSize() const = 0; }; class TTrivialNormalizerTask : public INormalizerTask { @@ -95,7 +97,9 @@ namespace NKikimr::NOlap { public: TTrivialNormalizerTask(const INormalizerChanges::TPtr& changes) : Changes(changes) - {} + { + AFL_VERIFY(Changes); + } void Start(const TNormalizationController& /* controller */, const TNormalizationContext& /*nCtx*/) override; }; @@ -136,9 +140,8 @@ namespace NKikimr::NOlap { virtual ENormalizerSequentialId GetType() const = 0; - const TString& GetName() const { - static TString name = ToString(GetType()); - return name; + TString GetName() const { + return ToString(GetType()); } ui32 GetSequentialId() const { diff --git a/ydb/core/tx/columnshard/normalizer/granule/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/granule/normalizer.cpp index 8f3dce081078..081b40f2379e 100644 --- a/ydb/core/tx/columnshard/normalizer/granule/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/granule/normalizer.cpp @@ -48,6 +48,10 @@ class TGranulesNormalizer::TNormalizerResult : public INormalizerChanges { return true; } + ui64 GetSize() const override { + return Chunks.size(); + } + static std::optional> Init(const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) { using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index cc741c33a586..09f19b3c0166 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -34,6 +34,10 @@ class TChunksNormalizer::TNormalizerResult : public INormalizerChanges { } return true; } + + ui64 GetSize() const override { + return Chunks.size(); + } }; class TRowsAndBytesChangesTask: public NConveyor::ITask { diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index 53b6643011d4..435083000889 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -34,6 +34,10 @@ class TBlobsRemovingResult : public INormalizerChanges { void ApplyOnComplete(const TNormalizationController& /* normController */) const override { RemovingAction->OnCompleteTxAfterRemoving(true); } + + ui64 GetSize() const override { + return Portions.size(); + } }; class TBlobsRemovingTask : public INormalizerTask { diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index 0ae36aa0251b..739715f44125 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -29,6 +29,10 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { } return true; } + + ui64 GetSize() const override { + return Portions.size(); + } }; bool TPortionsNormalizer::CheckPortion(const NColumnShard::TTablesManager&, const TPortionInfo& portionInfo) const { diff --git a/ydb/core/tx/columnshard/normalizer/tables/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/tables/normalizer.cpp index 2c8e11bad9be..63bd4c027edf 100644 --- a/ydb/core/tx/columnshard/normalizer/tables/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/tables/normalizer.cpp @@ -30,6 +30,10 @@ class TRemovedTablesNormalizer::TNormalizerResult : public INormalizerChanges { return true; } + ui64 GetSize() const override { + return PathIds.size(); + } + static std::optional> Init(NTabletFlatExecutor::TTransactionContext& txc) { using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/schemeshard/ut_olap/ya.make b/ydb/core/tx/schemeshard/ut_olap/ya.make index 1393af86188b..8a526327efb5 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ya.make +++ b/ydb/core/tx/schemeshard/ut_olap/ya.make @@ -22,6 +22,7 @@ PEERDIR( ydb/core/tx ydb/core/tx/columnshard ydb/core/tx/columnshard/test_helper + ydb/core/tx/columnshard/hooks/testing ydb/core/tx/schemeshard/ut_helpers ydb/library/yql/public/udf/service/exception_policy )