sys view for optimizer debug and control

ydb/core/sys_view/common/schema.cpp

@@ -217,9 +217,11 @@ class TSystemViewResolver : public ISystemViewResolver {
         RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName);
         RegisterOlapStoreSystemView<Schema::PrimaryIndexPortionStats>(StorePrimaryIndexPortionStatsName);
         RegisterOlapStoreSystemView<Schema::PrimaryIndexGranuleStats>(StorePrimaryIndexGranuleStatsName);
+        RegisterOlapStoreSystemView<Schema::PrimaryIndexOptimizerStats>(StorePrimaryIndexOptimizerStatsName);
         RegisterColumnTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName);
         RegisterColumnTableSystemView<Schema::PrimaryIndexPortionStats>(TablePrimaryIndexPortionStatsName);
         RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName);
+        RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName);
 
         RegisterSystemView<Schema::TopPartitions>(TopPartitions1MinuteName);
         RegisterSystemView<Schema::TopPartitions>(TopPartitions1HourName);

ydb/core/sys_view/common/schema.h

@@ -34,9 +34,11 @@ constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute";
 constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats";
 constexpr TStringBuf StorePrimaryIndexPortionStatsName = "store_primary_index_portion_stats";
 constexpr TStringBuf StorePrimaryIndexGranuleStatsName = "store_primary_index_granule_stats";
+constexpr TStringBuf StorePrimaryIndexOptimizerStatsName = "store_primary_index_optimizer_stats";
 constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats";
 constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_stats";
 constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats";
+constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats";
 
 constexpr TStringBuf TopPartitions1MinuteName = "top_partitions_one_minute";
 constexpr TStringBuf TopPartitions1HourName = "top_partitions_one_hour";
@@ -557,6 +559,33 @@ struct Schema : NIceDb::Schema {
         >;
     };
 
+    struct PrimaryIndexOptimizerStats: Table<14> {
+        struct PathId: Column<1, NScheme::NTypeIds::Uint64> {};
+        struct TabletId: Column<2, NScheme::NTypeIds::Uint64> {};
+        struct TaskId: Column<3, NScheme::NTypeIds::Uint64> {};
+        struct HostName: Column<4, NScheme::NTypeIds::Utf8> {};
+        struct NodeId: Column<5, NScheme::NTypeIds::Uint64> {};
+        struct Start: Column<6, NScheme::NTypeIds::Utf8> {};
+        struct Finish: Column<7, NScheme::NTypeIds::Utf8> {};
+        struct Details: Column<8, NScheme::NTypeIds::Utf8> {};
+        struct Category: Column<9, NScheme::NTypeIds::Uint64> {};
+        struct Weight: Column<10, NScheme::NTypeIds::Int64> {};
+
+        using TKey = TableKey<PathId, TabletId, TaskId>;
+        using TColumns = TableColumns<
+            PathId,
+            TabletId,
+            TaskId,
+            HostName,
+            NodeId,
+            Start,
+            Finish,
+            Details,
+            Category,
+            Weight
+        >;
+    };
+
 };
 
 bool MaybeSystemViewPath(const TVector<TString>& path);

ydb/core/tx/columnshard/common/snapshot.cpp

@@ -1,5 +1,7 @@
 #include "snapshot.h"
 #include <ydb/core/tx/columnshard/common/protos/snapshot.pb.h>
+
+#include <library/cpp/json/writer/json_value.h>
 #include <util/string/builder.h>
 
 namespace NKikimr::NOlap {
@@ -8,6 +10,13 @@ TString TSnapshot::DebugString() const {
     return TStringBuilder() << "plan_step=" << PlanStep << ";tx_id=" << TxId << ";";
 }
 
+NJson::TJsonValue TSnapshot::DebugJson() const {
+    NJson::TJsonValue result = NJson::JSON_MAP;
+    result.InsertValue("plan_step", PlanStep);
+    result.InsertValue("tx_id", TxId);
+    return result;
+}
+
 NKikimrColumnShardProto::TSnapshot TSnapshot::SerializeToProto() const {
     NKikimrColumnShardProto::TSnapshot result;
     SerializeToProto(result);

ydb/core/tx/columnshard/common/snapshot.h

@@ -9,6 +9,10 @@ namespace NKikimrColumnShardProto {
 class TSnapshot;
 }
 
+namespace NJson {
+class TJsonValue;
+};
+
 namespace NKikimr::NOlap {
 
 class TSnapshot {
@@ -84,6 +88,7 @@ class TSnapshot {
     TString SerializeToString() const;
 
     TString DebugString() const;
+    NJson::TJsonValue DebugJson() const;
 };
 
 } // namespace NKikimr::NOlap

ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp

@@ -22,7 +22,7 @@ NKikimr::TConclusionStatus TMetadataFromStore::DoFillMetadata(const NColumnShard
         auto pathInfos = logsIndex->GetTables(fromPathId, toPathId);
         for (auto&& pathInfo : pathInfos) {
             if (pathIds.emplace(pathInfo->GetPathId()).second) {
-                metadata->IndexGranules.emplace_back(NAbstract::TGranuleMetaView(*pathInfo, metadata->IsDescSorted()));
+                metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
             }
         }
     }
@@ -50,7 +50,7 @@ NKikimr::TConclusionStatus TMetadataFromTable::DoFillMetadata(const NColumnShard
             if (!pathInfo) {
                 continue;
             }
-            metadata->IndexGranules.emplace_back(NAbstract::TGranuleMetaView(*pathInfo, metadata->IsDescSorted()));
+            metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
             break;
         }
     }

ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "granule_view.h"
 #include <ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h>
 #include <ydb/core/tx/columnshard/engines/reader/common/description.h>
 
@@ -7,19 +8,27 @@ namespace NKikimr::NOlap::NReader::NSysView::NAbstract {
 class IMetadataFiller {
 private:
     virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const = 0;
+
+    virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
+        return NAbstract::TGranuleMetaView(granule, reverse);
+    }
 public:
     virtual ~IMetadataFiller() = default;
 
     TConclusionStatus FillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const {
         return DoFillMetadata(shard, metadata, read);
     }
+
+    NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
+        return DoBuildGranuleView(granule, reverse);
+    }
+
 };
 
 class TMetadataFromStore: public IMetadataFiller {
 protected:
     virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const override;
 public:
-
 };
 
 class TMetadataFromTable: public IMetadataFiller {

ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <ydb/core/tx/columnshard/engines/storage/granule.h>
+#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
 
 namespace NKikimr::NOlap::NReader::NSysView::NAbstract {
 
@@ -8,6 +9,7 @@ class TGranuleMetaView {
     using TPortions = std::deque<std::shared_ptr<TPortionInfo>>;
     YDB_READONLY(ui64, PathId, 0);
     YDB_READONLY_DEF(TPortions, Portions);
+    YDB_READONLY_DEF(std::vector<NStorageOptimizer::TTaskDescription>, OptimizerTasks);
 public:
     TGranuleMetaView(const TGranuleMeta& granule, const bool reverse)
         : PathId(granule.GetPathId())
@@ -26,6 +28,14 @@ class TGranuleMetaView {
         }
     }
 
+    void FillOptimizerTasks(const TGranuleMeta& granule, const bool reverse) {
+        OptimizerTasks = granule.GetOptimizerTasksDescription();
+        std::sort(OptimizerTasks.begin(), OptimizerTasks.end());
+        if (reverse) {
+            std::reverse(OptimizerTasks.begin(), OptimizerTasks.end());
+        }
+    }
+
     bool operator<(const TGranuleMetaView& item) const {
         return PathId < item.PathId;
     }

ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp

@@ -0,0 +1,40 @@
+#include "optimizer.h"
+#include <ydb/core/formats/arrow/switch/switch_type.h>
+#include <ydb/core/tx/columnshard/blobs_action/common/const.h>
+#include <ydb/core/tx/columnshard/engines/reader/abstract/read_context.h>
+#include <util/system/hostname.h>
+
+namespace NKikimr::NOlap::NReader::NSysView::NOptimizer {
+
+bool TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, NAbstract::TGranuleMetaView& granule) const {
+    for (auto&& i : granule.GetOptimizerTasks()) {
+        NArrow::Append<arrow::UInt64Type>(*builders[0], granule.GetPathId());
+        NArrow::Append<arrow::UInt64Type>(*builders[1], ReadMetadata->TabletId);
+        NArrow::Append<arrow::UInt64Type>(*builders[2], i.GetTaskId());
+        NArrow::Append<arrow::StringType>(*builders[3], HostNameField);
+        NArrow::Append<arrow::UInt64Type>(*builders[4], NActors::TActivationContext::AsActorContext().SelfID.NodeId());
+        NArrow::Append<arrow::StringType>(*builders[5], arrow::util::string_view(i.GetStart().data(), i.GetStart().size()));
+        NArrow::Append<arrow::StringType>(*builders[6], arrow::util::string_view(i.GetFinish().data(), i.GetFinish().size()));
+        NArrow::Append<arrow::StringType>(*builders[7], arrow::util::string_view(i.GetDetails().data(), i.GetDetails().size()));
+        NArrow::Append<arrow::UInt64Type>(*builders[8], i.GetWeightCategory());
+        NArrow::Append<arrow::Int64Type>(*builders[9], i.GetWeight());
+    }
+    return false;
+}
+
+std::unique_ptr<TScanIteratorBase> TReadStatsMetadata::StartScan(const std::shared_ptr<TReadContext>& readContext) const {
+    return std::make_unique<TStatsIterator>(readContext->GetReadMetadataPtrVerifiedAs<TReadStatsMetadata>());
+}
+
+std::vector<std::pair<TString, NKikimr::NScheme::TTypeInfo>> TReadStatsMetadata::GetKeyYqlSchema() const {
+    return GetColumns(TStatsIterator::StatsSchema, TStatsIterator::StatsSchema.KeyColumns);
+}
+
+std::shared_ptr<NAbstract::TReadStatsMetadata> TConstructor::BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const {
+    auto* index = self->GetIndexOptional();
+    return std::make_shared<TReadStatsMetadata>(index ? index->CopyVersionedIndexPtr() : nullptr, self->TabletID(),
+        IsReverse ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC,
+        read.GetProgram(), index ? index->GetVersionedIndex().GetLastSchema() : nullptr, read.GetSnapshot());
+}
+
+}

ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h

@@ -0,0 +1,90 @@
+#pragma once
+#include <ydb/core/sys_view/common/schema.h>
+#include <ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h>
+#include <ydb/core/tx/columnshard/engines/reader/sys_view/constructor/constructor.h>
+#include <util/system/hostname.h>
+
+namespace NKikimr::NOlap::NReader::NSysView::NOptimizer {
+
+class TConstructor: public TStatScannerConstructor<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats> {
+private:
+    using TBase = TStatScannerConstructor<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats>;
+protected:
+    virtual std::shared_ptr<NAbstract::TReadStatsMetadata> BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const override;
+
+public:
+    using TBase::TBase;
+};
+
+struct TReadStatsMetadata: public NAbstract::TReadStatsMetadata {
+private:
+    using TBase = NAbstract::TReadStatsMetadata;
+    using TSysViewSchema = NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats;
+public:
+    using TBase::TBase;
+
+    virtual std::unique_ptr<TScanIteratorBase> StartScan(const std::shared_ptr<TReadContext>& readContext) const override;
+    virtual std::vector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const override;
+};
+
+class TStatsIterator : public NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats> {
+private:
+    const std::string HostNameField = HostName();
+    using TBase = NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats>;
+    virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override {
+        return granule.GetOptimizerTasks().size();
+    }
+    virtual bool AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, NAbstract::TGranuleMetaView& granule) const override;
+public:
+    using TBase::TBase;
+};
+
+class TMetadataFromStore: public NAbstract::TMetadataFromStore {
+protected:
+    virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
+        NAbstract::TGranuleMetaView result(granule, reverse);
+        result.FillOptimizerTasks(granule, reverse);
+        return result;
+    }
+public:
+
+};
+
+class TMetadataFromTable: public NAbstract::TMetadataFromTable {
+protected:
+    virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
+        NAbstract::TGranuleMetaView result(granule, reverse);
+        result.FillOptimizerTasks(granule, reverse);
+        return result;
+    }
+public:
+
+};
+
+class TStoreSysViewPolicy: public NAbstract::ISysViewPolicy {
+protected:
+    virtual std::unique_ptr<IScannerConstructor> DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override {
+        return std::make_unique<TConstructor>(snapshot, itemsLimit, reverse);
+    }
+    virtual std::shared_ptr<NAbstract::IMetadataFiller> DoCreateMetadataFiller() const override {
+        return std::make_shared<TMetadataFromStore>();
+    }
+public:
+    static const inline TFactory::TRegistrator<TStoreSysViewPolicy> Registrator = TFactory::TRegistrator<TStoreSysViewPolicy>(TString(::NKikimr::NSysView::StorePrimaryIndexOptimizerStatsName));
+
+};
+
+class TTableSysViewPolicy: public NAbstract::ISysViewPolicy {
+protected:
+    virtual std::unique_ptr<IScannerConstructor> DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override {
+        return std::make_unique<TConstructor>(snapshot, itemsLimit, reverse);
+    }
+    virtual std::shared_ptr<NAbstract::IMetadataFiller> DoCreateMetadataFiller() const override {
+        return std::make_shared<TMetadataFromTable>();
+    }
+public:
+    static const inline TFactory::TRegistrator<TTableSysViewPolicy> Registrator = TFactory::TRegistrator<TTableSysViewPolicy>(TString(::NKikimr::NSysView::TablePrimaryIndexOptimizerStatsName));
+
+};
+
+}

ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/ya.make

@@ -0,0 +1,12 @@
+LIBRARY()
+
+PEERDIR(
+    ydb/core/tx/columnshard/engines/reader/sys_view/abstract
+)
+
+SRCS(
+    GLOBAL optimizer.cpp
+)
+
+END()
+

ydb/core/tx/columnshard/engines/reader/sys_view/ya.make

@@ -6,6 +6,7 @@ PEERDIR(
     ydb/core/tx/columnshard/engines/reader/sys_view/chunks
     ydb/core/tx/columnshard/engines/reader/sys_view/constructor
     ydb/core/tx/columnshard/engines/reader/sys_view/granules
+    ydb/core/tx/columnshard/engines/reader/sys_view/optimizer
 )
 
 SRCS(

ydb/core/tx/columnshard/engines/storage/granule.cpp

@@ -150,10 +150,6 @@ TGranuleMeta::TGranuleMeta(const ui64 pathId, const TGranulesStorage& owner, con
 
 }
 
-bool TGranuleMeta::InCompaction() const {
-    return Activity.contains(EActivity::GeneralCompaction);
-}
-
 std::shared_ptr<TPortionInfo> TGranuleMeta::UpsertPortionOnLoad(TPortionInfo&& portion) {
     auto portionId = portion.GetPortionId();
     auto emplaceInfo = Portions.emplace(portionId, std::make_shared<TPortionInfo>(std::move(portion)));

ydb/core/tx/columnshard/engines/storage/granule.h

@@ -162,6 +162,10 @@ class TGranuleMeta: TNonCopyable {
         ActualizationIndex->RefreshTiering(tiering, context);
     }
 
+    std::vector<NStorageOptimizer::TTaskDescription> GetOptimizerTasksDescription() const {
+        return OptimizerPlanner->GetTasksDescription();
+    }
+
     void RefreshScheme() {
         NActualizer::TAddExternalContext context(HasAppData() ? AppDataVerified().TimeProvider->Now() : TInstant::Now(), Portions);
         ActualizationIndex->RefreshScheme(context);
@@ -250,8 +254,6 @@ class TGranuleMeta: TNonCopyable {
         }
     }
 
-    bool InCompaction() const;
-
     bool IsErasable() const {
         return Activity.empty() && Portions.empty();
     }