diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index b8670a27380b..5bc2946deb45 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -378,7 +378,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl << " ttl settings: " << tableProto.GetTtlSettings() << " at tablet " << TabletID()); - TTableInfo::TTableVersionInfo tableVerProto; + NKikimrTxColumnShard::TTableVersionInfo tableVerProto; tableVerProto.SetPathId(pathId); // check schema changed @@ -439,7 +439,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP << " ttl settings: " << alterProto.GetTtlSettings() << " at tablet " << TabletID()); - TTableInfo::TTableVersionInfo tableVerProto; + NKikimrTxColumnShard::TTableVersionInfo tableVerProto; if (alterProto.HasSchemaPreset()) { tableVerProto.SetSchemaPresetId(alterProto.GetSchemaPreset().GetId()); TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index dd0a1426678e..e7d3ca6ba0c8 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -37,13 +37,18 @@ bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionConte } } json.InsertValue("tables_count", Tables.size()); - json.InsertValue("presets_count", SchemaPresets.size()); + json.InsertValue("presets_count", SchemaPresetsIds.size()); json.InsertValue("to_drop_count", PathsToDrop.size()); return true; } bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { + using TTableVersionsInfo = TVersionedSchema; + + THashMap schemaPresets; + THashMap tableVersions; { + TMemoryProfileGuard g("TTablesManager/InitFromDB::Tables"); auto rowset = db.Table().Select(); if (!rowset.IsReady()) { return false; @@ -57,6 +62,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { if (table.IsDropped()) { PathsToDrop.insert(table.GetPathId()); } + + AFL_VERIFY(tableVersions.emplace(table.GetPathId(), TTableVersionsInfo()).second); AFL_VERIFY(Tables.emplace(table.GetPathId(), std::move(table)).second); if (!rowset.Next()) { @@ -67,6 +74,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { bool isFakePresetOnly = true; { + TMemoryProfileGuard g("TTablesManager/InitFromDB::SchemaPresets"); auto rowset = db.Table().Select(); if (!rowset.IsReady()) { return false; @@ -82,7 +90,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { Y_VERIFY_S(preset.GetName() == "default", "Preset name: " + preset.GetName()); isFakePresetOnly = false; } - AFL_VERIFY(SchemaPresets.emplace(preset.GetId(), preset).second); + AFL_VERIFY(schemaPresets.emplace(preset.GetId(), preset).second); + AFL_VERIFY(SchemaPresetsIds.emplace(preset.GetId()).second); if (!rowset.Next()) { return false; } @@ -90,6 +99,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { } { + TMemoryProfileGuard g("TTablesManager/InitFromDB::Versions"); auto rowset = db.Table().Select(); if (!rowset.IsReady()) { return false; @@ -101,13 +111,14 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { Y_ABORT_UNLESS(Tables.contains(pathId)); NOlap::TSnapshot version( rowset.GetValue(), - rowset.GetValue()); + rowset.GetValue()); - auto& table = Tables.at(pathId); - TTableInfo::TTableVersionInfo versionInfo; + auto& table = Tables[pathId]; + auto& versionsInfo = tableVersions[pathId]; + NKikimrTxColumnShard::TTableVersionInfo versionInfo; Y_ABORT_UNLESS(versionInfo.ParseFromString(rowset.GetValue())); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "load_table_version")("path_id", pathId)("snapshot", version)("version", versionInfo.HasSchema() ? versionInfo.GetSchema().GetVersion() : -1); - Y_ABORT_UNLESS(SchemaPresets.contains(versionInfo.GetSchemaPresetId())); + Y_ABORT_UNLESS(schemaPresets.contains(versionInfo.GetSchemaPresetId())); if (!table.IsDropped()) { auto& ttlSettings = versionInfo.GetTtlSettings(); @@ -120,7 +131,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { } } } - table.AddVersion(version, versionInfo); + table.AddVersion(version); + versionsInfo.AddVersion(version, versionInfo); if (!rowset.Next()) { return false; } @@ -128,6 +140,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { } { + TMemoryProfileGuard g("TTablesManager/InitFromDB::PresetVersions"); auto rowset = db.Table().Select(); if (!rowset.IsReady()) { return false; @@ -135,8 +148,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { while (!rowset.EndOfSet()) { const ui32 id = rowset.GetValue(); - Y_ABORT_UNLESS(SchemaPresets.contains(id)); - auto& preset = SchemaPresets.at(id); + Y_ABORT_UNLESS(schemaPresets.contains(id)); + auto& preset = schemaPresets[id]; NOlap::TSnapshot version( rowset.GetValue(), rowset.GetValue()); @@ -151,19 +164,20 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { } } - for (const auto& [id, preset] : SchemaPresets) { + TMemoryProfileGuard g("TTablesManager/InitFromDB::Other"); + for (const auto& [id, preset] : schemaPresets) { if (isFakePresetOnly) { Y_ABORT_UNLESS(id == 0); } else { Y_ABORT_UNLESS(id > 0); } - for (const auto& [version, schemaInfo] : preset.GetVersions()) { + for (const auto& [version, schemaInfo] : preset.GetVersionsById()) { if (schemaInfo.HasSchema()) { AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "index_schema")("preset_id", id)("snapshot", version)("version", schemaInfo.GetSchema().GetVersion()); if (!PrimaryIndex) { - PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schemaInfo.GetSchema()); + PrimaryIndex = std::make_unique(TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema()); } else { - PrimaryIndex->RegisterSchemaVersion(version, schemaInfo.GetSchema()); + PrimaryIndex->RegisterSchemaVersion(preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema()); } } } @@ -199,7 +213,7 @@ bool TTablesManager::IsReadyForWrite(const ui64 pathId) const { } bool TTablesManager::HasPreset(const ui32 presetId) const { - return SchemaPresets.contains(presetId); + return SchemaPresetsIds.contains(presetId); } const TTableInfo& TTablesManager::GetTable(const ui64 pathId) const { @@ -211,8 +225,7 @@ ui64 TTablesManager::GetMemoryUsage() const { ui64 memory = Tables.size() * sizeof(TTableInfo) + PathsToDrop.size() * sizeof(ui64) + - Ttl.PathsCount() * sizeof(TTtl::TDescription) + - SchemaPresets.size() * sizeof(TSchemaPreset); + Ttl.PathsCount() * sizeof(TTtl::TDescription); if (PrimaryIndex) { memory += PrimaryIndex->MemoryUsage(); } @@ -220,7 +233,8 @@ ui64 TTablesManager::GetMemoryUsage() const { } void TTablesManager::DropTable(const ui64 pathId, const NOlap::TSnapshot& version, NIceDb::TNiceDb& db) { - auto& table = Tables.at(pathId); + AFL_VERIFY(Tables.contains(pathId)); + auto& table = Tables[pathId]; table.SetDropVersion(version); PathsToDrop.insert(pathId); Ttl.DropPathTtl(pathId); @@ -228,9 +242,8 @@ void TTablesManager::DropTable(const ui64 pathId, const NOlap::TSnapshot& versio } void TTablesManager::DropPreset(const ui32 presetId, const NOlap::TSnapshot& version, NIceDb::TNiceDb& db) { - auto& preset = SchemaPresets.at(presetId); - Y_ABORT_UNLESS(preset.GetName() != "default", "Cannot drop the default preset"); - preset.SetDropVersion(version); + AFL_VERIFY(SchemaPresetsIds.contains(presetId)); + SchemaPresetsIds.erase(presetId); Schema::SaveSchemaPresetDropVersion(db, presetId, version); } @@ -247,17 +260,16 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { } bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db) { - if (SchemaPresets.contains(schemaPreset.GetId())) { + if (SchemaPresetsIds.contains(schemaPreset.GetId())) { return false; } + SchemaPresetsIds.emplace(schemaPreset.GetId()); Schema::SaveSchemaPresetInfo(db, schemaPreset.GetId(), schemaPreset.GetName()); - SchemaPresets.emplace(schemaPreset.GetId(), schemaPreset); return true; } void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db) { - Y_ABORT_UNLESS(SchemaPresets.contains(presetId)); - auto preset = SchemaPresets.at(presetId); + Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId)); TSchemaPreset::TSchemaPresetVersionInfo versionInfo; versionInfo.SetId(presetId); @@ -265,9 +277,7 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho versionInfo.SetSinceTxId(version.GetTxId()); *versionInfo.MutableSchema() = schema; - auto& schemaPreset = SchemaPresets.at(presetId); Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); - schemaPreset.AddVersion(version, versionInfo); if (versionInfo.HasSchema()) { if (!PrimaryIndex) { PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema); @@ -283,21 +293,21 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho } } -void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const TTableInfo::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr& manager) { +void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr& manager) { auto it = Tables.find(pathId); AFL_VERIFY(it != Tables.end()); auto& table = it->second; if (versionInfo.HasSchemaPresetId()) { - Y_ABORT_UNLESS(SchemaPresets.contains(versionInfo.GetSchemaPresetId())); + Y_ABORT_UNLESS(SchemaPresetsIds.contains(versionInfo.GetSchemaPresetId())); } else if (versionInfo.HasSchema()) { TSchemaPreset fakePreset; - if (SchemaPresets.empty()) { + if (SchemaPresetsIds.empty()) { TSchemaPreset fakePreset; Y_ABORT_UNLESS(RegisterSchemaPreset(fakePreset, db)); AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db); } else { - Y_ABORT_UNLESS(SchemaPresets.contains(fakePreset.GetId())); + Y_ABORT_UNLESS(SchemaPresetsIds.contains(fakePreset.GetId())); AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db); } } @@ -314,7 +324,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& } } Schema::SaveTableVersionInfo(db, pathId, version, versionInfo); - table.AddVersion(version, versionInfo); + table.AddVersion(version); } TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId) @@ -332,7 +342,7 @@ bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, co const auto& itTable = Tables.find(pathId); AFL_VERIFY(itTable != Tables.end())("problem", "No schema for path")("path_id", pathId); for (auto&& tableVersion : itTable->second.GetVersions()) { - NColumnShard::Schema::EraseTableVersionInfo(db, pathId, tableVersion.first); + NColumnShard::Schema::EraseTableVersionInfo(db, pathId, tableVersion); } return true; } diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 31f2fdd7808f..60e1a5f66dbd 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -13,43 +13,40 @@ namespace NKikimr::NColumnShard { -template +template class TVersionedSchema { -protected: - std::optional DropVersion; - TMap Versions; - +private: + TMap Versions; + TMap VersionsById; + TMap MinVersionById; public: - bool IsDropped() const { - return DropVersion.has_value(); - } - bool IsEmpty() const { - return Versions.empty(); + return VersionsById.empty(); } - void SetDropVersion(const NOlap::TSnapshot& version) { - DropVersion = version; + const TMap& GetVersionsById() const { + return VersionsById; } - const TMap& GetVersions() const { - return Versions; + NOlap::TSnapshot GetMinVersionForId(const ui64 sVersion) const { + auto it = MinVersionById.find(sVersion); + Y_ABORT_UNLESS(it != MinVersionById.end()); + return it->second; } - const TSchemaProto& GetVersion(const NOlap::TSnapshot& version) const { - const TSchemaProto* result = nullptr; - for (auto ver : Versions) { - if (ver.first > version) { - break; - } - result = &ver.second; + void AddVersion(const NOlap::TSnapshot& snapshot, const TVersionData& versionInfo) { + ui64 ssVersion = 0; + if (versionInfo.HasSchema()) { + ssVersion = versionInfo.GetSchema().GetVersion(); } - Y_ABORT_UNLESS(!!result); - return *result; - } + VersionsById.emplace(ssVersion, versionInfo); + Y_ABORT_UNLESS(Versions.emplace(snapshot, ssVersion).second); - void AddVersion(const NOlap::TSnapshot& version, const TSchemaProto& versionInfo) { - Versions[version] = versionInfo; + if (MinVersionById.contains(ssVersion)) { + MinVersionById.emplace(ssVersion, std::min(snapshot, MinVersionById.at(ssVersion))); + } else { + MinVersionById.emplace(ssVersion, snapshot); + } } }; @@ -80,22 +77,16 @@ class TSchemaPreset : public TVersionedSchema(); } Y_ABORT_UNLESS(!Id || Name == "default", "Unsupported preset at load time"); - - if (rowset.template HaveValue() && - rowset.template HaveValue()) - { - DropVersion.emplace(rowset.template GetValue(), - rowset.template GetValue()); - } return true; } }; -class TTableInfo : public TVersionedSchema { +class TTableInfo { public: - using TTableVersionInfo = NKikimrTxColumnShard::TTableVersionInfo; ui64 PathId; TString TieringUsage; + std::optional DropVersion; + YDB_READONLY_DEF(TSet, Versions); public: const TString& GetTieringUsage() const { @@ -107,10 +98,26 @@ class TTableInfo : public TVersionedSchema Tables; - THashMap SchemaPresets; + THashSet SchemaPresetsIds; THashSet PathsToDrop; TTtl Ttl; std::unique_ptr PrimaryIndex; @@ -163,8 +170,8 @@ class TTablesManager { return Tables; } - const THashMap& GetSchemaPresets() const { - return SchemaPresets; + const THashSet& GetSchemaPresets() const { + return SchemaPresetsIds; } bool HasPrimaryIndex() const { @@ -233,7 +240,7 @@ class TTablesManager { bool RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db); void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db); - void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const TTableInfo::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr& manager); + void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr& manager); bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); };