From 710487130bb9df5954452cf8360c9be812b0081a Mon Sep 17 00:00:00 2001 From: kungurtsev Date: Tue, 27 Aug 2024 16:37:21 +0200 Subject: [PATCH] 24-3 Report index sizes by type (#8325) --- ydb/core/tablet_flat/flat_database.cpp | 3 ++- ydb/core/tablet_flat/flat_dbase_misc.h | 2 +- ydb/core/tablet_flat/flat_executor.cpp | 4 +++- ydb/core/tablet_flat/flat_executor_counters.h | 2 ++ ydb/core/tablet_flat/flat_table.cpp | 18 ++++++++++++++---- ydb/core/tablet_flat/flat_table_stats.h | 3 ++- ydb/core/tablet_flat/ut/ut_db_iface.cpp | 3 ++- 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/ydb/core/tablet_flat/flat_database.cpp b/ydb/core/tablet_flat/flat_database.cpp index b87ee8056d59..ef50fab90283 100644 --- a/ydb/core/tablet_flat/flat_database.cpp +++ b/ydb/core/tablet_flat/flat_database.cpp @@ -494,7 +494,8 @@ ui64 TDatabase::GetTableMemOpsCount(ui32 tableId) const { } ui64 TDatabase::GetTableIndexSize(ui32 tableId) const { - return Require(tableId)->Stat().Parts.IndexBytes; + const auto& partStats = Require(tableId)->Stat().Parts; + return partStats.FlatIndexBytes + partStats.BTreeIndexBytes; } ui64 TDatabase::GetTableSearchHeight(ui32 tableId) const { diff --git a/ydb/core/tablet_flat/flat_dbase_misc.h b/ydb/core/tablet_flat/flat_dbase_misc.h index 1163eb05f9ba..4c3fe35fd252 100644 --- a/ydb/core/tablet_flat/flat_dbase_misc.h +++ b/ydb/core/tablet_flat/flat_dbase_misc.h @@ -14,7 +14,7 @@ namespace NTable { void Describe(IOutputStream &out) const noexcept { - const ui64 sys = Parts.IndexBytes + Parts.ByKeyBytes + Parts.OtherBytes; + const ui64 sys = Parts.FlatIndexBytes + Parts.BTreeIndexBytes + Parts.ByKeyBytes + Parts.OtherBytes; out << "DBase{" << Tables << "t " << Parts.PartsCount << "p" diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 81bd63b86c72..152064a6d907 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -3590,7 +3590,9 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) { { /* Memory consumption of common for leader and follower components */ Counters->Simple()[TExecutorCounters::DB_WARM_BYTES].Set(dbCounters.MemTableBytes); Counters->Simple()[TExecutorCounters::DB_META_BYTES].Set(Stats->PacksMetaBytes); - Counters->Simple()[TExecutorCounters::DB_INDEX_BYTES].Set(dbCounters.Parts.IndexBytes); + Counters->Simple()[TExecutorCounters::DB_FLAT_INDEX_BYTES].Set(dbCounters.Parts.FlatIndexBytes); + Counters->Simple()[TExecutorCounters::DB_B_TREE_INDEX_BYTES].Set(dbCounters.Parts.BTreeIndexBytes); + Counters->Simple()[TExecutorCounters::DB_INDEX_BYTES].Set(dbCounters.Parts.FlatIndexBytes + dbCounters.Parts.BTreeIndexBytes); Counters->Simple()[TExecutorCounters::DB_OTHER_BYTES].Set(dbCounters.Parts.OtherBytes); Counters->Simple()[TExecutorCounters::DB_BYKEY_BYTES].Set(dbCounters.Parts.ByKeyBytes); Counters->Simple()[TExecutorCounters::USED_TABLET_MEMORY].Set(UsedTabletMemory); diff --git a/ydb/core/tablet_flat/flat_executor_counters.h b/ydb/core/tablet_flat/flat_executor_counters.h index 914fdad59995..22a1dc7b03b2 100644 --- a/ydb/core/tablet_flat/flat_executor_counters.h +++ b/ydb/core/tablet_flat/flat_executor_counters.h @@ -62,6 +62,8 @@ namespace NTabletFlatExecutor { XX(CONSUMED_STORAGE, "ConsumedStorage") \ XX(CONSUMED_MEMORY, "ConsumedMemory") \ XX(COMPACTION_READ_IN_FLY, "CompactionReadInFly") \ + XX(DB_FLAT_INDEX_BYTES, "DbFlatIndexBytes") \ + XX(DB_B_TREE_INDEX_BYTES, "DbBTreeIndexBytes") \ // don't change order! #define FLAT_EXECUTOR_CUMULATIVE_COUNTERS_MAP(XX) \ diff --git a/ydb/core/tablet_flat/flat_table.cpp b/ydb/core/tablet_flat/flat_table.cpp index 354bccbcaefc..ca0b9e58533d 100644 --- a/ydb/core/tablet_flat/flat_table.cpp +++ b/ydb/core/tablet_flat/flat_table.cpp @@ -1415,7 +1415,11 @@ void TTable::SetTableObserver(TIntrusivePtr ptr) noexcept void TPartStats::Add(const TPartView& partView) { PartsCount += 1; - IndexBytes += partView->IndexesRawSize; + if (partView->IndexPages.HasBTree()) { + BTreeIndexBytes += partView->IndexesRawSize; + } else { + FlatIndexBytes += partView->IndexesRawSize; + } ByKeyBytes += partView->ByKey ? partView->ByKey->Raw.size() : 0; PlainBytes += partView->Stat.Bytes; CodedBytes += partView->Stat.Coded; @@ -1434,7 +1438,11 @@ void TPartStats::Add(const TPartView& partView) bool TPartStats::Remove(const TPartView& partView) { NUtil::SubSafe(PartsCount, ui64(1)); - NUtil::SubSafe(IndexBytes, partView->IndexesRawSize); + if (partView->IndexPages.HasBTree()) { + NUtil::SubSafe(BTreeIndexBytes, partView->IndexesRawSize); + } else { + NUtil::SubSafe(FlatIndexBytes, partView->IndexesRawSize); + } NUtil::SubSafe(ByKeyBytes, partView->ByKey ? partView->ByKey->Raw.size() : 0); NUtil::SubSafe(PlainBytes, partView->Stat.Bytes); NUtil::SubSafe(CodedBytes, partView->Stat.Coded); @@ -1463,7 +1471,8 @@ bool TPartStats::Remove(const TPartView& partView) TPartStats& TPartStats::operator+=(const TPartStats& rhs) { PartsCount += rhs.PartsCount; - IndexBytes += rhs.IndexBytes; + FlatIndexBytes += rhs.FlatIndexBytes; + BTreeIndexBytes += rhs.BTreeIndexBytes; OtherBytes += rhs.OtherBytes; ByKeyBytes += rhs.ByKeyBytes; PlainBytes += rhs.PlainBytes; @@ -1480,7 +1489,8 @@ TPartStats& TPartStats::operator+=(const TPartStats& rhs) TPartStats& TPartStats::operator-=(const TPartStats& rhs) { NUtil::SubSafe(PartsCount, rhs.PartsCount); - NUtil::SubSafe(IndexBytes, rhs.IndexBytes); + NUtil::SubSafe(FlatIndexBytes, rhs.FlatIndexBytes); + NUtil::SubSafe(BTreeIndexBytes, rhs.BTreeIndexBytes); NUtil::SubSafe(OtherBytes, rhs.OtherBytes); NUtil::SubSafe(ByKeyBytes, rhs.ByKeyBytes); NUtil::SubSafe(PlainBytes, rhs.PlainBytes); diff --git a/ydb/core/tablet_flat/flat_table_stats.h b/ydb/core/tablet_flat/flat_table_stats.h index 172c777ac3dd..4d735d3fccdc 100644 --- a/ydb/core/tablet_flat/flat_table_stats.h +++ b/ydb/core/tablet_flat/flat_table_stats.h @@ -8,7 +8,8 @@ namespace NTable { struct TPartStats { ui64 PartsCount = 0; /* Total used TPart units in db */ - ui64 IndexBytes = 0; + ui64 FlatIndexBytes = 0; + ui64 BTreeIndexBytes = 0; ui64 OtherBytes = 0; /* Other metadata and sys. indexes */ ui64 ByKeyBytes = 0; ui64 PlainBytes = 0; /* Plain data pages size */ diff --git a/ydb/core/tablet_flat/ut/ut_db_iface.cpp b/ydb/core/tablet_flat/ut/ut_db_iface.cpp index b82252553f46..39fb1473ebe0 100644 --- a/ydb/core/tablet_flat/ut/ut_db_iface.cpp +++ b/ydb/core/tablet_flat/ut/ut_db_iface.cpp @@ -276,7 +276,8 @@ Y_UNIT_TEST_SUITE(DBase) { UNIT_ASSERT(me->Counters().Parts.RowsErase == 0); UNIT_ASSERT(me->Counters().Parts.PartsCount == 0); UNIT_ASSERT(me->Counters().Parts.PlainBytes == 0); - UNIT_ASSERT(me->Counters().Parts.IndexBytes == 0); + UNIT_ASSERT(me->Counters().Parts.FlatIndexBytes == 0); + UNIT_ASSERT(me->Counters().Parts.BTreeIndexBytes == 0); UNIT_ASSERT(me->Counters().Parts.OtherBytes == 0); }