From ac2677f33884ad0ca9fafc94f4d80405b8405762 Mon Sep 17 00:00:00 2001 From: Sofya Novozhiova Date: Mon, 5 Feb 2024 19:31:35 +0000 Subject: [PATCH 1/3] Prepare common library for locks --- ydb/core/client/locks_ut.cpp | 2 +- .../engine/minikql/minikql_engine_host.cpp | 2 +- .../kqp/runtime/kqp_stream_lookup_worker.h | 2 +- ydb/core/sys_view/common/schema.h | 2 +- ydb/core/tx/columnshard/columnshard__init.cpp | 10 + ydb/core/tx/columnshard/columnshard_impl.cpp | 1 + ydb/core/tx/columnshard/columnshard_impl.h | 49 +++- ydb/core/tx/columnshard/columnshard_schema.h | 46 +++- .../tx/columnshard/transactions/locks_db.cpp | 5 + .../tx/columnshard/transactions/locks_db.h | 22 ++ ydb/core/tx/columnshard/transactions/ya.make | 7 +- .../build_and_wait_dependencies_unit.cpp | 3 +- .../tx/datashard/datashard__engine_host.cpp | 2 +- .../datashard_active_transaction.cpp | 2 +- .../datashard/datashard_active_transaction.h | 2 +- ydb/core/tx/datashard/datashard_dep_tracker.h | 2 +- .../datashard/datashard_direct_transaction.h | 2 +- ydb/core/tx/datashard/datashard_impl.h | 2 +- ydb/core/tx/datashard/datashard_kqp.cpp | 2 +- ydb/core/tx/datashard/datashard_kqp.h | 2 +- ydb/core/tx/datashard/datashard_locks_db.cpp | 205 ----------------- ydb/core/tx/datashard/datashard_locks_db.h | 67 +++--- ydb/core/tx/datashard/datashard_user_table.h | 2 - ydb/core/tx/datashard/datashard_ut_locks.cpp | 2 +- .../datashard/datashard_write_operation.cpp | 2 +- .../tx/datashard/datashard_write_operation.h | 2 +- ydb/core/tx/datashard/key_conflicts.cpp | 2 +- ydb/core/tx/datashard/operation.h | 2 +- ydb/core/tx/datashard/read_iterator.h | 2 +- ydb/core/tx/datashard/ya.make | 7 +- .../datashard_locks.cpp => locks/locks.cpp} | 8 +- .../datashard_locks.h => locks/locks.h} | 0 ydb/core/tx/locks/locks_db.cpp | 4 + ydb/core/tx/locks/locks_db.h | 209 ++++++++++++++++++ .../tx/{datashard => locks}/range_treap.cpp | 0 .../tx/{datashard => locks}/range_treap.h | 0 .../{datashard => locks}/range_treap_ut.cpp | 0 .../tx/{datashard => locks}/range_tree_base.h | 0 ydb/core/tx/{datashard => locks}/sys_tables.h | 0 .../time_counters.cpp} | 3 +- .../time_counters.h} | 0 ydb/core/tx/locks/ya.make | 19 ++ ydb/core/tx/scheme_board/cache.cpp | 2 +- ydb/core/tx/scheme_cache/scheme_cache.h | 2 +- ydb/core/tx/tx_proxy/proxy.h | 2 +- ydb/services/metadata/manager/abstract.h | 2 +- 46 files changed, 422 insertions(+), 289 deletions(-) create mode 100644 ydb/core/tx/columnshard/transactions/locks_db.cpp create mode 100644 ydb/core/tx/columnshard/transactions/locks_db.h rename ydb/core/tx/{datashard/datashard_locks.cpp => locks/locks.cpp} (99%) rename ydb/core/tx/{datashard/datashard_locks.h => locks/locks.h} (100%) create mode 100644 ydb/core/tx/locks/locks_db.cpp create mode 100644 ydb/core/tx/locks/locks_db.h rename ydb/core/tx/{datashard => locks}/range_treap.cpp (100%) rename ydb/core/tx/{datashard => locks}/range_treap.h (100%) rename ydb/core/tx/{datashard => locks}/range_treap_ut.cpp (100%) rename ydb/core/tx/{datashard => locks}/range_tree_base.h (100%) rename ydb/core/tx/{datashard => locks}/sys_tables.h (100%) rename ydb/core/tx/{datashard/datashard_counters.cpp => locks/time_counters.cpp} (61%) rename ydb/core/tx/{datashard/datashard_counters.h => locks/time_counters.h} (100%) create mode 100644 ydb/core/tx/locks/ya.make diff --git a/ydb/core/client/locks_ut.cpp b/ydb/core/client/locks_ut.cpp index 2617461edaf7..575ed97b7722 100644 --- a/ydb/core/client/locks_ut.cpp +++ b/ydb/core/client/locks_ut.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index bbddd3310ca3..f720c8780e12 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.h b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.h index b4bf0f47ef8a..adf44c052c19 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.h +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace NKikimr { diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index a3ef043fadc8..8eb8fe8fdd90 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -3,7 +3,7 @@ #include "path.h" #include -#include +#include namespace NKikimr { namespace NSysView { diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 3e11ed9e9217..285ffd11b69d 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -4,6 +4,7 @@ #include "columnshard_schema.h" #include "hooks/abstract/abstract.h" #include +#include #include #include @@ -177,6 +178,15 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) } } } + { + TMemoryProfileGuard g("TTxInit/LocksDB"); + if (txc.DB.GetScheme().GetTableInfo(Schema::Locks::TableId)) { + TColumnShardLocksDb locksDb(*Self, txc); + if (!Self->SysLocks.Load(locksDb)) { + return false; + } + } + } Self->UpdateInsertTableCounters(); Self->UpdateIndexCounters(); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 5d5df25e8fe9..9ac408efd80a 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -93,6 +93,7 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , ScanCounters("Scan") , WritesMonitor(*this) , NormalizerController(StoragesManager, SubscribeCounters) + , SysLocks(this) { TabletCountersPtr.reset(new TProtobufTabletCounters< ESimpleCounters_descriptor, diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 00e72583c23a..55f26e3aa50d 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -25,8 +25,11 @@ #include #include #include +#include #include +#include + namespace NKikimr::NOlap { class TCleanupColumnEngineChanges; class TTTLColumnEngineChanges; @@ -206,10 +209,6 @@ class TColumnShard TabletCounters->Cumulative()[counter].Increment(num); } - void IncCounter(NColumnShard::EPercentileCounters counter, const TDuration& latency) const { - TabletCounters->Percentile()[counter].IncrementFor(latency.MicroSeconds()); - } - void ActivateTiering(const ui64 pathId, const TString& useTiering, const bool onTabletInit = false); void OnTieringModified(); public: @@ -220,6 +219,41 @@ class TColumnShard None /* "none" */ }; + void IncCounter(NColumnShard::EPercentileCounters counter, const TDuration& latency) const { + TabletCounters->Percentile()[counter].IncrementFor(latency.MicroSeconds()); + } + + void IncCounter(NDataShard::ESimpleCounters counter, ui64 num = 1) const { + TabletCounters->Simple()[counter].Add(num); + } + + // For systable + void IncCounter(NDataShard::ECumulativeCounters counter, ui64 num = 1) const { + TabletCounters->Cumulative()[counter].Increment(num); + } + + void IncCounter(NDataShard::EPercentileCounters counter, ui64 num) const { + TabletCounters->Percentile()[counter].IncrementFor(num); + } + + void IncCounter(NDataShard::EPercentileCounters counter, const TDuration& latency) const { + TabletCounters->Percentile()[counter].IncrementFor(latency.MilliSeconds()); + } + + inline TRowVersion LastCompleteTxVersion() const { + return TRowVersion(LastCompletedStep, LastCompletedTxId); + } + + ui32 Generation() const { return Executor()->Generation(); } + + bool IsUserTable(const TTableId&) const { + return false; + } + + const THashMap &GetUserTables() const { + return Default>(); + } + private: void OverloadWriteFail(const EOverloadStatus overloadReason, const NEvWrite::TWriteData& writeData, std::unique_ptr&& event, const TActorContext& ctx); EOverloadStatus CheckOverloaded(const ui64 tableId) const; @@ -281,6 +315,9 @@ class TColumnShard } } +public: + TTabletCountersBase* TabletCounters; + private: std::unique_ptr ProgressTxController; std::unique_ptr OperationsManager; @@ -360,6 +397,8 @@ class TColumnShard TWriteId LastWriteId = TWriteId{0}; ui64 LastPlannedStep = 0; ui64 LastPlannedTxId = 0; + ui64 LastCompletedStep = 0; + ui64 LastCompletedTxId = 0; ui64 LastExportNo = 0; ui64 OwnerPathId = 0; @@ -386,7 +425,6 @@ class TColumnShard TTablesManager TablesManager; std::shared_ptr Tiers; std::unique_ptr TabletCountersPtr; - TTabletCountersBase* TabletCounters; std::unique_ptr PipeClientCache; std::unique_ptr InsertTable; std::shared_ptr SubscribeCounters; @@ -412,6 +450,7 @@ class TColumnShard TLimits Limits; TCompactionLimits CompactionLimits; NOlap::TNormalizationController NormalizerController; + NDataShard::TSysLocks SysLocks; void TryRegisterMediatorTimeCast(); void UnregisterMediatorTimeCast(); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 1690fa0bfec7..f7a1b20b0bb4 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -38,7 +38,11 @@ struct Schema : NIceDb::Schema { ColumnsTableId, CountersTableId, OperationsTableId, - IndexesTableId + IndexesTableId, + LocksTableId, + LockRangesTableId, + LockConflictsTableId, + LockVolatileDependenciesTableId }; enum class ETierTables: ui32 { @@ -305,6 +309,46 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns; }; + struct Locks : Table { + struct LockId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct LockNodeId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct Generation : Column<3, NScheme::NTypeIds::Uint32> {}; + struct Counter : Column<4, NScheme::NTypeIds::Uint64> {}; + struct CreateTimestamp : Column<5, NScheme::NTypeIds::Uint64> {}; + struct Flags : Column<6, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + + struct LockRanges : Table { + struct LockId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct RangeId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct PathOwnerId : Column<3, NScheme::NTypeIds::Uint64> {}; + struct LocalPathId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct Flags : Column<5, NScheme::NTypeIds::Uint64> {}; + struct Data : Column<6, NScheme::NTypeIds::String> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + + struct LockConflicts : Table { + struct LockId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct ConflictId : Column<2, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + + struct LockVolatileDependencies : Table { + struct LockId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct TxId : Column<2, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + using TTables = SchemaTables< Value, TxInfo, diff --git a/ydb/core/tx/columnshard/transactions/locks_db.cpp b/ydb/core/tx/columnshard/transactions/locks_db.cpp new file mode 100644 index 000000000000..d3196304b86c --- /dev/null +++ b/ydb/core/tx/columnshard/transactions/locks_db.cpp @@ -0,0 +1,5 @@ +#include "locks_db.h" + +namespace NKikimr::NColumnShard { + +} diff --git a/ydb/core/tx/columnshard/transactions/locks_db.h b/ydb/core/tx/columnshard/transactions/locks_db.h new file mode 100644 index 000000000000..ed8d237d4fcf --- /dev/null +++ b/ydb/core/tx/columnshard/transactions/locks_db.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include + + +namespace NKikimr::NColumnShard { + +class TColumnShardLocksDb : public NLocks::TShardLocksDb { +private: + using TBase = NLocks::TShardLocksDb; + +public: + using TBase::TBase; + + void PersistRemoveLock(ui64 lockId) override { + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId).Delete(); + HasChanges_ = true; + } +}; + +} diff --git a/ydb/core/tx/columnshard/transactions/ya.make b/ydb/core/tx/columnshard/transactions/ya.make index 17f70379bb89..d7798e5ef33d 100644 --- a/ydb/core/tx/columnshard/transactions/ya.make +++ b/ydb/core/tx/columnshard/transactions/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( tx_controller.cpp + locks_db.cpp ) PEERDIR( @@ -9,12 +10,6 @@ PEERDIR( ydb/core/tx/data_events ) -IF (OS_WINDOWS) - CFLAGS( - -DKIKIMR_DISABLE_S3_OPS - ) -ENDIF() - YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp b/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp index 8f41a1da2a0a..ebdb2d18ede3 100644 --- a/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp +++ b/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp @@ -1,8 +1,9 @@ #include "datashard_impl.h" -#include "datashard_counters.h" #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#include + namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index ad98ab4c1c7e..06e8caca6d84 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -2,7 +2,7 @@ #include "datashard_impl.h" #include "datashard_user_db.h" #include "datashard__engine_host.h" -#include "sys_tables.h" +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp index 8d9b8bf2011b..b106458c24d8 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.cpp +++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp @@ -2,11 +2,11 @@ #include "datashard_active_transaction.h" #include "datashard_kqp.h" -#include "datashard_locks.h" #include "datashard_impl.h" #include "datashard_failpoints.h" #include "key_conflicts.h" +#include #include namespace NKikimr { diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index dd1870362b7b..f2fe16498158 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -1,7 +1,7 @@ #pragma once #include "datashard.h" -#include "datashard_locks.h" +#include #include "datashard__engine_host.h" #include "operation.h" diff --git a/ydb/core/tx/datashard/datashard_dep_tracker.h b/ydb/core/tx/datashard/datashard_dep_tracker.h index 4c5a3746b02c..b48fcd3baaad 100644 --- a/ydb/core/tx/datashard/datashard_dep_tracker.h +++ b/ydb/core/tx/datashard/datashard_dep_tracker.h @@ -3,7 +3,7 @@ #include "datashard.h" #include "datashard_user_table.h" #include "datashard_active_transaction.h" -#include "range_treap.h" +#include #include diff --git a/ydb/core/tx/datashard/datashard_direct_transaction.h b/ydb/core/tx/datashard/datashard_direct_transaction.h index 85cacda5f840..6bdb518e351b 100644 --- a/ydb/core/tx/datashard/datashard_direct_transaction.h +++ b/ydb/core/tx/datashard/datashard_direct_transaction.h @@ -1,7 +1,7 @@ #pragma once #include "datashard_impl.h" -#include "datashard_locks.h" +#include #include "operation.h" #include diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 7720260dce60..5a9b8d100387 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -1,7 +1,6 @@ #pragma once #include "datashard.h" -#include "datashard_locks.h" #include "datashard_trans_queue.h" #include "datashard_outreadset.h" #include "datashard_pipeline.h" @@ -29,6 +28,7 @@ #include #include #include +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp index efd84c1ba70a..71ed5ff64239 100644 --- a/ydb/core/tx/datashard/datashard_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_kqp.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_kqp.h b/ydb/core/tx/datashard/datashard_kqp.h index 12c230e92f12..62b5d860f40e 100644 --- a/ydb/core/tx/datashard/datashard_kqp.h +++ b/ydb/core/tx/datashard/datashard_kqp.h @@ -4,11 +4,11 @@ #include "operation.h" #include "key_validator.h" -#include "datashard_locks.h" #include "datashard_user_db.h" #include #include +#include #include diff --git a/ydb/core/tx/datashard/datashard_locks_db.cpp b/ydb/core/tx/datashard/datashard_locks_db.cpp index 9285acb6df44..88af1639b092 100644 --- a/ydb/core/tx/datashard/datashard_locks_db.cpp +++ b/ydb/core/tx/datashard/datashard_locks_db.cpp @@ -2,209 +2,4 @@ namespace NKikimr::NDataShard { -bool TDataShardLocksDb::Load(TVector& rows) { - using Schema = TDataShard::Schema; - - NIceDb::TNiceDb db(DB); - - rows.clear(); - - // Load locks - THashMap lockIndex; - { - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - while (!rowset.EndOfSet()) { - auto& lock = rows.emplace_back(); - lock.LockId = rowset.GetValue(); - lock.LockNodeId = rowset.GetValue(); - lock.Generation = rowset.GetValue(); - lock.Counter = rowset.GetValue(); - lock.CreateTs = rowset.GetValue(); - lock.Flags = rowset.GetValue(); - lockIndex[lock.LockId] = rows.size() - 1; - if (!rowset.Next()) { - return false; - } - } - } - - // Load ranges - { - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - while (!rowset.EndOfSet()) { - auto lockId = rowset.GetValue(); - auto it = lockIndex.find(lockId); - if (it != lockIndex.end()) { - auto& lock = rows[it->second]; - auto& range = lock.Ranges.emplace_back(); - range.RangeId = rowset.GetValue(); - range.TableId.OwnerId = rowset.GetValue(); - range.TableId.LocalPathId = rowset.GetValue(); - range.Flags = rowset.GetValue(); - range.Data = rowset.GetValue(); - } - if (!rowset.Next()) { - return false; - } - } - } - - // Load conflicts - { - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - while (!rowset.EndOfSet()) { - auto lockId = rowset.GetValue(); - auto it = lockIndex.find(lockId); - if (it != lockIndex.end()) { - auto& lock = rows[it->second]; - lock.Conflicts.push_back(rowset.GetValue()); - } - if (!rowset.Next()) { - return false; - } - } - } - - // Load volatile dependencies - if (db.HaveTable()) { - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - while (!rowset.EndOfSet()) { - auto lockId = rowset.GetValue(); - auto it = lockIndex.find(lockId); - if (it != lockIndex.end()) { - auto& lock = rows[it->second]; - auto txId = rowset.GetValue(); - lock.VolatileDependencies.push_back(txId); - } - if (!rowset.Next()) { - return false; - } - } - } - - return true; -} - -bool TDataShardLocksDb::MayAddLock(ui64 lockId) { - for (auto& pr : Self.GetUserTables()) { - auto tid = pr.second->LocalTid; - // We cannot start a new lockId if it has any uncompacted data - if (DB.HasTxData(tid, lockId)) { - return false; - } - } - - return true; -} - -void TDataShardLocksDb::PersistAddLock(ui64 lockId, ui32 lockNodeId, ui32 generation, ui64 counter, ui64 createTs, ui64 flags) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId).Update( - NIceDb::TUpdate(lockNodeId), - NIceDb::TUpdate(generation), - NIceDb::TUpdate(counter), - NIceDb::TUpdate(createTs), - NIceDb::TUpdate(flags)); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistLockCounter(ui64 lockId, ui64 counter) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId).Update( - NIceDb::TUpdate(counter)); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistRemoveLock(ui64 lockId) { - // We remove lock changes unless it's managed by volatile tx manager - bool isVolatile = Self.GetVolatileTxManager().FindByCommitTxId(lockId); - if (!isVolatile) { - for (auto& pr : Self.GetUserTables()) { - auto tid = pr.second->LocalTid; - // Removing the lock also removes any uncommitted data - if (DB.HasOpenTx(tid, lockId)) { - DB.RemoveTx(tid, lockId); - Self.GetConflictsCache().GetTableCache(tid).RemoveUncommittedWrites(lockId, DB); - } - } - } - - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId).Delete(); - HasChanges_ = true; - - if (!isVolatile) { - Self.ScheduleRemoveLockChanges(lockId); - } -} - -void TDataShardLocksDb::PersistAddRange(ui64 lockId, ui64 rangeId, const TPathId& tableId, ui64 flags, const TString& data) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, rangeId).Update( - NIceDb::TUpdate(tableId.OwnerId), - NIceDb::TUpdate(tableId.LocalPathId), - NIceDb::TUpdate(flags), - NIceDb::TUpdate(data)); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistRangeFlags(ui64 lockId, ui64 rangeId, ui64 flags) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, rangeId).Update( - NIceDb::TUpdate(flags)); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistRemoveRange(ui64 lockId, ui64 rangeId) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, rangeId).Delete(); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistAddConflict(ui64 lockId, ui64 otherLockId) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, otherLockId).Update(); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistRemoveConflict(ui64 lockId, ui64 otherLockId) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, otherLockId).Delete(); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistAddVolatileDependency(ui64 lockId, ui64 txId) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, txId).Update(); - HasChanges_ = true; -} - -void TDataShardLocksDb::PersistRemoveVolatileDependency(ui64 lockId, ui64 txId) { - using Schema = TDataShard::Schema; - NIceDb::TNiceDb db(DB); - db.Table().Key(lockId, txId).Delete(); - HasChanges_ = true; -} - } // namespace NKikimr::NDataShard diff --git a/ydb/core/tx/datashard/datashard_locks_db.h b/ydb/core/tx/datashard/datashard_locks_db.h index 1aba7176ec94..f91e21bf09f2 100644 --- a/ydb/core/tx/datashard/datashard_locks_db.h +++ b/ydb/core/tx/datashard/datashard_locks_db.h @@ -1,47 +1,40 @@ #pragma once +#include + #include "datashard_impl.h" namespace NKikimr::NDataShard { -class TDataShardLocksDb - : public ILocksDb -{ -public: - TDataShardLocksDb(TDataShard& self, TTransactionContext& txc) - : Self(self) - , DB(txc.DB) - { } +class TDataShardLocksDb : public NLocks::TShardLocksDb { +private: + using TBase = NLocks::TShardLocksDb; - bool HasChanges() const { - return HasChanges_; +public: + using TBase::TBase; + + void PersistRemoveLock(ui64 lockId) override { + // We remove lock changes unless it's managed by volatile tx manager + bool isVolatile = Self.GetVolatileTxManager().FindByCommitTxId(lockId); + if (!isVolatile) { + for (auto& pr : Self.GetUserTables()) { + auto tid = pr.second->LocalTid; + // Removing the lock also removes any uncommitted data + if (DB.HasOpenTx(tid, lockId)) { + DB.RemoveTx(tid, lockId); + Self.GetConflictsCache().GetTableCache(tid).RemoveUncommittedWrites(lockId, DB); + } + } + } + + using Schema = TDataShard::Schema; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId).Delete(); + HasChanges_ = true; + + if (!isVolatile) { + Self.ScheduleRemoveLockChanges(lockId); + } } - - bool Load(TVector& rows) override; - - bool MayAddLock(ui64 lockId) override; - - // Persist adding/removing a lock info - void PersistAddLock(ui64 lockId, ui32 lockNodeId, ui32 generation, ui64 counter, ui64 createTs, ui64 flags = 0) override; - void PersistLockCounter(ui64 lockId, ui64 counter) override; - void PersistRemoveLock(ui64 lockId) override; - - // Persist adding/removing info on locked ranges - void PersistAddRange(ui64 lockId, ui64 rangeId, const TPathId& tableId, ui64 flags = 0, const TString& data = {}) override; - void PersistRangeFlags(ui64 lockId, ui64 rangeId, ui64 flags) override; - void PersistRemoveRange(ui64 lockId, ui64 rangeId) override; - - // Persist a conflict, i.e. this lock must break some other lock on commit - void PersistAddConflict(ui64 lockId, ui64 otherLockId) override; - void PersistRemoveConflict(ui64 lockId, ui64 otherLockId) override; - - // Persist volatile dependencies, i.e. which undecided transactions must be waited for on commit - void PersistAddVolatileDependency(ui64 lockId, ui64 txId) override; - void PersistRemoveVolatileDependency(ui64 lockId, ui64 txId) override; - -private: - TDataShard& Self; - NTable::TDatabase& DB; - bool HasChanges_ = false; }; } // namespace NKikimr::NDataShard diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 40a53145c998..a1e2927411ad 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -1,7 +1,5 @@ #pragma once -#include "datashard.h" - #include #include #include diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index 029af4fb409e..c42cbb11a53d 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -1,5 +1,5 @@ #include "defs.h" -#include "datashard_locks.h" +#include #include #include diff --git a/ydb/core/tx/datashard/datashard_write_operation.cpp b/ydb/core/tx/datashard/datashard_write_operation.cpp index 335dc5c6ff79..de99492cfef5 100644 --- a/ydb/core/tx/datashard/datashard_write_operation.cpp +++ b/ydb/core/tx/datashard/datashard_write_operation.cpp @@ -2,7 +2,7 @@ #include "datashard_write_operation.h" #include "datashard_kqp.h" -#include "datashard_locks.h" +#include #include "datashard_impl.h" #include "datashard_failpoints.h" diff --git a/ydb/core/tx/datashard/datashard_write_operation.h b/ydb/core/tx/datashard/datashard_write_operation.h index 1ae40f39c02f..345e30687c5e 100644 --- a/ydb/core/tx/datashard/datashard_write_operation.h +++ b/ydb/core/tx/datashard/datashard_write_operation.h @@ -1,7 +1,7 @@ #pragma once #include "datashard_impl.h" -#include "datashard_locks.h" +#include #include "datashard__engine_host.h" #include "datashard_user_db.h" #include "operation.h" diff --git a/ydb/core/tx/datashard/key_conflicts.cpp b/ydb/core/tx/datashard/key_conflicts.cpp index a252a1d9212c..bdf24c7b6dc5 100644 --- a/ydb/core/tx/datashard/key_conflicts.cpp +++ b/ydb/core/tx/datashard/key_conflicts.cpp @@ -1,5 +1,5 @@ #include "key_conflicts.h" -#include "sys_tables.h" +#include namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/operation.h b/ydb/core/tx/datashard/operation.h index 7c7e5bf13769..fe7ffbe2f7d0 100644 --- a/ydb/core/tx/datashard/operation.h +++ b/ydb/core/tx/datashard/operation.h @@ -2,7 +2,7 @@ #include "defs.h" #include "datashard.h" -#include "datashard_locks.h" +#include #include "datashard_outreadset.h" #include "datashard_snapshots.h" #include "execution_unit_kind.h" diff --git a/ydb/core/tx/datashard/read_iterator.h b/ydb/core/tx/datashard/read_iterator.h index 3ce26ec01220..a42653089eb2 100644 --- a/ydb/core/tx/datashard/read_iterator.h +++ b/ydb/core/tx/datashard/read_iterator.h @@ -1,7 +1,7 @@ #pragma once #include "datashard.h" -#include "datashard_locks.h" +#include #include #include diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 49541b974798..12f02b8ed5da 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -74,12 +74,9 @@ SRCS( datashard_change_receiving.cpp datashard_change_sender_activation.cpp datashard_change_sending.cpp - datashard_counters.cpp datashard_loans.cpp datashard_locks_db.cpp datashard_locks_db.h - datashard_locks.h - datashard_locks.cpp datashard_split_dst.cpp datashard_split_src.cpp datashard_switch_mvcc_state.cpp @@ -127,7 +124,7 @@ SRCS( datashard_repl_offsets_client.cpp datashard_repl_offsets_server.cpp datashard_subdomain_path_id.cpp - datashard_write_operation.cpp + datashard_write_operation.cpp datashard_txs.h datashard.cpp datashard.h @@ -186,7 +183,6 @@ SRCS( remove_locks.cpp range_avl_tree.cpp range_ops.cpp - range_treap.cpp read_iterator.h restore_unit.cpp setup_sys_locks.h @@ -245,6 +241,7 @@ PEERDIR( ydb/core/tablet ydb/core/tablet_flat ydb/core/tx/long_tx_service/public + ydb/core/tx/locks ydb/core/util ydb/core/wrappers ydb/core/ydb_convert diff --git a/ydb/core/tx/datashard/datashard_locks.cpp b/ydb/core/tx/locks/locks.cpp similarity index 99% rename from ydb/core/tx/datashard/datashard_locks.cpp rename to ydb/core/tx/locks/locks.cpp index a7e5aecbaa5c..c7630948207f 100644 --- a/ydb/core/tx/datashard/datashard_locks.cpp +++ b/ydb/core/tx/locks/locks.cpp @@ -1,10 +1,12 @@ -#include "datashard_locks.h" -#include "datashard_impl.h" -#include "datashard_counters.h" +#include "locks.h" +#include "time_counters.h" + #include #include +#include + namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/datashard_locks.h b/ydb/core/tx/locks/locks.h similarity index 100% rename from ydb/core/tx/datashard/datashard_locks.h rename to ydb/core/tx/locks/locks.h diff --git a/ydb/core/tx/locks/locks_db.cpp b/ydb/core/tx/locks/locks_db.cpp new file mode 100644 index 000000000000..1b015e3e377b --- /dev/null +++ b/ydb/core/tx/locks/locks_db.cpp @@ -0,0 +1,4 @@ +#include "locks_db.h" + +namespace NKikimr::NLocks { +} diff --git a/ydb/core/tx/locks/locks_db.h b/ydb/core/tx/locks/locks_db.h new file mode 100644 index 000000000000..1edf54e3f563 --- /dev/null +++ b/ydb/core/tx/locks/locks_db.h @@ -0,0 +1,209 @@ +#pragma once + +#include "locks.h" +#include +#include + +namespace NKikimr::NLocks { + +template +class TShardLocksDb : public NKikimr::NDataShard::ILocksDb { +public: + TShardLocksDb(TShard& self, NTabletFlatExecutor::TTransactionContext& txc) + : Self(self) + , DB(txc.DB) + { } + + bool HasChanges() const { + return HasChanges_; + } + + bool Load(TVector& rows) override { + using Schema = TSchemaDescription; + + NIceDb::TNiceDb db(DB); + + rows.clear(); + + // Load locks + THashMap lockIndex; + { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + auto& lock = rows.emplace_back(); + lock.LockId = rowset.template GetValue(); + lock.LockNodeId = rowset.template GetValue(); + lock.Generation = rowset.template GetValue(); + lock.Counter = rowset.template GetValue(); + lock.CreateTs = rowset.template GetValue(); + lock.Flags = rowset.template GetValue(); + lockIndex[lock.LockId] = rows.size() - 1; + if (!rowset.Next()) { + return false; + } + } + } + + // Load ranges + { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + auto lockId = rowset.template GetValue(); + auto it = lockIndex.find(lockId); + if (it != lockIndex.end()) { + auto& lock = rows[it->second]; + auto& range = lock.Ranges.emplace_back(); + range.RangeId = rowset.template GetValue(); + range.TableId.OwnerId = rowset.template GetValue(); + range.TableId.LocalPathId = rowset.template GetValue(); + range.Flags = rowset.template GetValue(); + range.Data = rowset.template GetValue(); + } + if (!rowset.Next()) { + return false; + } + } + } + + // Load conflicts + { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + auto lockId = rowset.template GetValue(); + auto it = lockIndex.find(lockId); + if (it != lockIndex.end()) { + auto& lock = rows[it->second]; + lock.Conflicts.push_back(rowset.template GetValue()); + } + if (!rowset.Next()) { + return false; + } + } + } + + // Load volatile dependencies + if (db.HaveTable()) { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + auto lockId = rowset.template GetValue(); + auto it = lockIndex.find(lockId); + if (it != lockIndex.end()) { + auto& lock = rows[it->second]; + auto txId = rowset.template GetValue(); + lock.VolatileDependencies.push_back(txId); + } + if (!rowset.Next()) { + return false; + } + } + } + + return true; + } + + bool MayAddLock(ui64 lockId) override { + for (auto& pr : Self.GetUserTables()) { + auto tid = pr.second->LocalTid; + // We cannot start a new lockId if it has any uncompacted data + if (DB.HasTxData(tid, lockId)) { + return false; + } + } + return true; + } + + void PersistAddLock(ui64 lockId, ui32 lockNodeId, ui32 generation, ui64 counter, ui64 createTs, ui64 flags = 0) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId).Update( + NIceDb::TUpdate(lockNodeId), + NIceDb::TUpdate(generation), + NIceDb::TUpdate(counter), + NIceDb::TUpdate(createTs), + NIceDb::TUpdate(flags)); + HasChanges_ = true; + } + + void PersistLockCounter(ui64 lockId, ui64 counter) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId).Update( + NIceDb::TUpdate(counter)); + HasChanges_ = true; + } + + // Persist adding/removing info on locked ranges + void PersistAddRange(ui64 lockId, ui64 rangeId, const TPathId& tableId, ui64 flags = 0, const TString& data = {}) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, rangeId).Update( + NIceDb::TUpdate(tableId.OwnerId), + NIceDb::TUpdate(tableId.LocalPathId), + NIceDb::TUpdate(flags), + NIceDb::TUpdate(data)); + HasChanges_ = true; + } + + void PersistRangeFlags(ui64 lockId, ui64 rangeId, ui64 flags) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, rangeId).Update( + NIceDb::TUpdate(flags)); + HasChanges_ = true; + } + + void PersistRemoveRange(ui64 lockId, ui64 rangeId) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, rangeId).Delete(); + HasChanges_ = true; + } + + // Persist a conflict, i.e. this lock must break some other lock on commit + void PersistAddConflict(ui64 lockId, ui64 otherLockId) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, otherLockId).Update(); + HasChanges_ = true; + } + + void PersistRemoveConflict(ui64 lockId, ui64 otherLockId) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, otherLockId).Delete(); + HasChanges_ = true; + } + + // Persist volatile dependencies, i.e. which undecided transactions must be waited for on commit + void PersistAddVolatileDependency(ui64 lockId, ui64 txId) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, txId).Update(); + HasChanges_ = true; + } + + void PersistRemoveVolatileDependency(ui64 lockId, ui64 txId) override { + using Schema = TSchemaDescription; + NIceDb::TNiceDb db(DB); + db.Table().Key(lockId, txId).Delete(); + HasChanges_ = true; + } + +protected: + TShard& Self; + NTable::TDatabase& DB; + bool HasChanges_ = false; +}; +} diff --git a/ydb/core/tx/datashard/range_treap.cpp b/ydb/core/tx/locks/range_treap.cpp similarity index 100% rename from ydb/core/tx/datashard/range_treap.cpp rename to ydb/core/tx/locks/range_treap.cpp diff --git a/ydb/core/tx/datashard/range_treap.h b/ydb/core/tx/locks/range_treap.h similarity index 100% rename from ydb/core/tx/datashard/range_treap.h rename to ydb/core/tx/locks/range_treap.h diff --git a/ydb/core/tx/datashard/range_treap_ut.cpp b/ydb/core/tx/locks/range_treap_ut.cpp similarity index 100% rename from ydb/core/tx/datashard/range_treap_ut.cpp rename to ydb/core/tx/locks/range_treap_ut.cpp diff --git a/ydb/core/tx/datashard/range_tree_base.h b/ydb/core/tx/locks/range_tree_base.h similarity index 100% rename from ydb/core/tx/datashard/range_tree_base.h rename to ydb/core/tx/locks/range_tree_base.h diff --git a/ydb/core/tx/datashard/sys_tables.h b/ydb/core/tx/locks/sys_tables.h similarity index 100% rename from ydb/core/tx/datashard/sys_tables.h rename to ydb/core/tx/locks/sys_tables.h diff --git a/ydb/core/tx/datashard/datashard_counters.cpp b/ydb/core/tx/locks/time_counters.cpp similarity index 61% rename from ydb/core/tx/datashard/datashard_counters.cpp rename to ydb/core/tx/locks/time_counters.cpp index 81c71ded164a..4c7d1b1ea05e 100644 --- a/ydb/core/tx/datashard/datashard_counters.cpp +++ b/ydb/core/tx/locks/time_counters.cpp @@ -1,5 +1,4 @@ -#include "datashard_counters.h" -#include "datashard_impl.h" +#include "time_counters.h" namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/datashard_counters.h b/ydb/core/tx/locks/time_counters.h similarity index 100% rename from ydb/core/tx/datashard/datashard_counters.h rename to ydb/core/tx/locks/time_counters.h diff --git a/ydb/core/tx/locks/ya.make b/ydb/core/tx/locks/ya.make new file mode 100644 index 000000000000..a1e6e543d3b4 --- /dev/null +++ b/ydb/core/tx/locks/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +SRCS( + locks.cpp + locks_db.cpp + time_counters.cpp + range_treap.cpp +) + + +PEERDIR( + ydb/core/protos + ydb/core/tablet_flat +) + +YQL_LAST_ABI_VERSION() + +END() + diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index c02f74e8f567..f3867aab2956 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -5,7 +5,7 @@ #include "monitorable_actor.h" #include "subscriber.h" -#include +#include #include #include #include diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 5ccea3f421a1..687a6e158e76 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h index e757bc3306ad..8ae80a9c1949 100644 --- a/ydb/core/tx/tx_proxy/proxy.h +++ b/ydb/core/tx/tx_proxy/proxy.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h index 132afb8a8a12..caba9204b622 100644 --- a/ydb/services/metadata/manager/abstract.h +++ b/ydb/services/metadata/manager/abstract.h @@ -3,7 +3,7 @@ #include "table_record.h" #include -#include +#include #include #include #include From 93fb56326ef49a3be3ca391823ba78d4b9c97b5b Mon Sep 17 00:00:00 2001 From: Sofya Novozhiova Date: Tue, 6 Feb 2024 07:46:12 +0000 Subject: [PATCH 2/3] Fix build andcorrect dept --- ydb/core/kqp/ut/opt/kqp_ne_ut.cpp | 1 + ydb/core/kqp/ut/scan/kqp_scan_ut.cpp | 1 + ydb/core/kqp/ut/scan/kqp_split_ut.cpp | 1 + ydb/core/tx/columnshard/columnshard_impl.h | 10 +- .../tx/columnshard/transactions/locks_db.h | 5 + ydb/core/tx/datashard/datashard_impl.h | 2 +- ydb/core/tx/datashard/datashard_locks_db.h | 11 +++ ydb/core/tx/datashard/datashard_ut_locks.cpp | 2 +- ydb/core/tx/datashard/range_avl_tree.h | 2 +- .../CMakeLists.darwin-arm64.txt | 86 ----------------- .../CMakeLists.darwin-x86_64.txt | 87 ------------------ .../CMakeLists.linux-aarch64.txt | 90 ------------------ .../CMakeLists.linux-x86_64.txt | 92 ------------------- .../datashard/ut_range_treap/CMakeLists.txt | 19 ---- .../CMakeLists.windows-x86_64.txt | 80 ---------------- ydb/core/tx/locks/locks.cpp | 7 +- ydb/core/tx/locks/locks.h | 8 +- ydb/core/tx/locks/locks_db.h | 11 --- .../ut_range_treap/ya.make | 2 +- ydb/core/tx/locks/ya.make | 4 + 20 files changed, 34 insertions(+), 487 deletions(-) delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-arm64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-x86_64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-aarch64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-x86_64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.txt delete mode 100644 ydb/core/tx/datashard/ut_range_treap/CMakeLists.windows-x86_64.txt rename ydb/core/tx/{datashard => locks}/ut_range_treap/ya.make (94%) diff --git a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp index 2c0067a1a755..5628244bcf25 100644 --- a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp b/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp index 093889290bb3..2f5ebd4cd674 100644 --- a/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/ydb/core/kqp/ut/scan/kqp_split_ut.cpp b/ydb/core/kqp/ut/scan/kqp_split_ut.cpp index 81e67c27ccb4..830cdc6ffd9b 100644 --- a/ydb/core/kqp/ut/scan/kqp_split_ut.cpp +++ b/ydb/core/kqp/ut/scan/kqp_split_ut.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 55f26e3aa50d..8bd78793ed56 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -28,8 +28,6 @@ #include #include -#include - namespace NKikimr::NOlap { class TCleanupColumnEngineChanges; class TTTLColumnEngineChanges; @@ -227,7 +225,7 @@ class TColumnShard TabletCounters->Simple()[counter].Add(num); } - // For systable + // For syslocks void IncCounter(NDataShard::ECumulativeCounters counter, ui64 num = 1) const { TabletCounters->Cumulative()[counter].Increment(num); } @@ -247,11 +245,7 @@ class TColumnShard ui32 Generation() const { return Executor()->Generation(); } bool IsUserTable(const TTableId&) const { - return false; - } - - const THashMap &GetUserTables() const { - return Default>(); + return true; } private: diff --git a/ydb/core/tx/columnshard/transactions/locks_db.h b/ydb/core/tx/columnshard/transactions/locks_db.h index ed8d237d4fcf..1df8fb957ff3 100644 --- a/ydb/core/tx/columnshard/transactions/locks_db.h +++ b/ydb/core/tx/columnshard/transactions/locks_db.h @@ -17,6 +17,11 @@ class TColumnShardLocksDb : public NLocks::TShardLocksDb().Key(lockId).Delete(); HasChanges_ = true; } + + bool MayAddLock(ui64) override { + return true; + } + }; } diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 5a9b8d100387..03c7124f4460 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -1586,7 +1586,7 @@ class TDataShard void AddUserTable(const TPathId& tableId, TUserTable::TPtr tableInfo) { TableInfos[tableId.LocalPathId] = tableInfo; - SysLocks.UpdateSchema(tableId, *tableInfo); + SysLocks.UpdateSchema(tableId, tableInfo->KeyColumnTypes); Pipeline.GetDepTracker().UpdateSchema(tableId, *tableInfo); } diff --git a/ydb/core/tx/datashard/datashard_locks_db.h b/ydb/core/tx/datashard/datashard_locks_db.h index f91e21bf09f2..0cf56cc43beb 100644 --- a/ydb/core/tx/datashard/datashard_locks_db.h +++ b/ydb/core/tx/datashard/datashard_locks_db.h @@ -35,6 +35,17 @@ class TDataShardLocksDb : public NLocks::TShardLocksDbLocalTid; + // We cannot start a new lockId if it has any uncompacted data + if (DB.HasTxData(tid, lockId)) { + return false; + } + } + return true; + } }; } // namespace NKikimr::NDataShard diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index c42cbb11a53d..bc6a251f36c1 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -152,7 +152,7 @@ namespace NTest { TmpLockVec.emplace_back(TCell::Make(TmpLock.SchemeShard)); TmpLockVec.emplace_back(TCell::Make(TmpLock.PathId)); - Locks.UpdateSchema(tableId.PathId, DataShard.TableInfos[tid]); + Locks.UpdateSchema(tableId.PathId, DataShard.TableInfos[tid].KeyColumnTypes); } // diff --git a/ydb/core/tx/datashard/range_avl_tree.h b/ydb/core/tx/datashard/range_avl_tree.h index 459c59711f71..c568b4b0ed73 100644 --- a/ydb/core/tx/datashard/range_avl_tree.h +++ b/ydb/core/tx/datashard/range_avl_tree.h @@ -1,6 +1,6 @@ #pragma once -#include "range_tree_base.h" +#include #include #include diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-arm64.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-arm64.txt deleted file mode 100644 index dcf8838cd46f..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-arm64.txt +++ /dev/null @@ -1,86 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_treap) -target_compile_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC - contrib-libs-cxxsupp - yutil - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_treap_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_treap - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_treap - TEST_TARGET - ydb-core-tx-datashard-ut_range_treap - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_treap - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_treap) diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index 89c959394de8..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,87 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_treap) -target_compile_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_treap_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_treap - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_treap - TEST_TARGET - ydb-core-tx-datashard-ut_range_treap - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_treap - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_treap) diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-aarch64.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-aarch64.txt deleted file mode 100644 index 10b1d8272c45..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,90 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_treap) -target_compile_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_treap_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_treap - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_treap - TEST_TARGET - ydb-core-tx-datashard-ut_range_treap - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_treap - cpp-malloc-jemalloc -) -vcs_info(ydb-core-tx-datashard-ut_range_treap) diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-x86_64.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-x86_64.txt deleted file mode 100644 index ce18da096f0f..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,92 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_treap) -target_compile_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_treap_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_treap - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_treap - TEST_TARGET - ydb-core-tx-datashard-ut_range_treap - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_treap - cpp-malloc-tcmalloc - libs-tcmalloc-no_percpu_cache -) -vcs_info(ydb-core-tx-datashard-ut_range_treap) diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.txt deleted file mode 100644 index d863ebd18067..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - include(CMakeLists.darwin-arm64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -endif() diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.windows-x86_64.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 66ad767b6592..000000000000 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,80 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_treap) -target_compile_options(ydb-core-tx-datashard-ut_range_treap PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_sources(ydb-core-tx-datashard-ut_range_treap PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_treap_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_treap - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_treap - TEST_TARGET - ydb-core-tx-datashard-ut_range_treap - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_treap - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_treap - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_treap) diff --git a/ydb/core/tx/locks/locks.cpp b/ydb/core/tx/locks/locks.cpp index c7630948207f..d9c68e29f64e 100644 --- a/ydb/core/tx/locks/locks.cpp +++ b/ydb/core/tx/locks/locks.cpp @@ -2,11 +2,8 @@ #include "time_counters.h" #include - #include -#include - namespace NKikimr { namespace NDataShard { @@ -610,11 +607,11 @@ void TLockLocker::RemoveLock(ui64 lockId, ILocksDb* db) { RemoveOneLock(lockId, db); } -void TLockLocker::UpdateSchema(const TPathId& tableId, const TUserTable& tableInfo) { +void TLockLocker::UpdateSchema(const TPathId& tableId, const TVector& keyColumnTypes) { TTableLocks::TPtr& table = Tables[tableId]; if (!table) table.Reset(new TTableLocks(tableId)); - table->UpdateKeyColumnsTypes(tableInfo.KeyColumnTypes); + table->UpdateKeyColumnsTypes(keyColumnTypes); } void TLockLocker::RemoveSchema(const TPathId& tableId) { diff --git a/ydb/core/tx/locks/locks.h b/ydb/core/tx/locks/locks.h index 2d95dc74e748..05b788594491 100644 --- a/ydb/core/tx/locks/locks.h +++ b/ydb/core/tx/locks/locks.h @@ -19,8 +19,6 @@ namespace NKikimr { namespace NDataShard { -struct TUserTable; - class ILocksDb { protected: ~ILocksDb() = default; @@ -577,7 +575,7 @@ class TLockLocker { }; } - void UpdateSchema(const TPathId& tableId, const TUserTable& tableInfo); + void UpdateSchema(const TPathId& tableId, const TVector& keyColumnTypes); void RemoveSchema(const TPathId& tableId); bool ForceShardLock(const TPathId& tableId) const; bool ForceShardLock(const TIntrusiveList& readTables) const; @@ -813,8 +811,8 @@ class TSysLocks { return Update->LockTxId; } - void UpdateSchema(const TPathId& tableId, const TUserTable& tableInfo) { - Locker.UpdateSchema(tableId, tableInfo); + void UpdateSchema(const TPathId& tableId, const TVector& keyColumnTypes) { + Locker.UpdateSchema(tableId, keyColumnTypes); } void RemoveSchema(const TPathId& tableId) { diff --git a/ydb/core/tx/locks/locks_db.h b/ydb/core/tx/locks/locks_db.h index 1edf54e3f563..1a78958b13bf 100644 --- a/ydb/core/tx/locks/locks_db.h +++ b/ydb/core/tx/locks/locks_db.h @@ -113,17 +113,6 @@ class TShardLocksDb : public NKikimr::NDataShard::ILocksDb { return true; } - bool MayAddLock(ui64 lockId) override { - for (auto& pr : Self.GetUserTables()) { - auto tid = pr.second->LocalTid; - // We cannot start a new lockId if it has any uncompacted data - if (DB.HasTxData(tid, lockId)) { - return false; - } - } - return true; - } - void PersistAddLock(ui64 lockId, ui32 lockNodeId, ui32 generation, ui64 counter, ui64 createTs, ui64 flags = 0) override { using Schema = TSchemaDescription; NIceDb::TNiceDb db(DB); diff --git a/ydb/core/tx/datashard/ut_range_treap/ya.make b/ydb/core/tx/locks/ut_range_treap/ya.make similarity index 94% rename from ydb/core/tx/datashard/ut_range_treap/ya.make rename to ydb/core/tx/locks/ut_range_treap/ya.make index bd8648affdb4..b6aba4f21439 100644 --- a/ydb/core/tx/datashard/ut_range_treap/ya.make +++ b/ydb/core/tx/locks/ut_range_treap/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(ydb/core/tx/datashard) +UNITTEST_FOR(ydb/core/tx/locks) FORK_SUBTESTS() diff --git a/ydb/core/tx/locks/ya.make b/ydb/core/tx/locks/ya.make index a1e6e543d3b4..afded9eea5e1 100644 --- a/ydb/core/tx/locks/ya.make +++ b/ydb/core/tx/locks/ya.make @@ -17,3 +17,7 @@ YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut_range_treap +) From a8416a50b2e8907737e541363b184ddabf24ead9 Mon Sep 17 00:00:00 2001 From: Sofya Novozhiova Date: Tue, 6 Feb 2024 09:37:40 +0000 Subject: [PATCH 3/3] Remove range_avl_tree --- ydb/core/tx/columnshard/columnshard__init.cpp | 9 + .../columnshard/columnshard__progress_tx.cpp | 10 + ydb/core/tx/columnshard/columnshard_impl.h | 9 +- ydb/core/tx/columnshard/columnshard_schema.h | 2 + ydb/core/tx/datashard/range_avl_tree.cpp | 0 ydb/core/tx/datashard/range_avl_tree.h | 721 ------------------ ydb/core/tx/datashard/range_avl_tree_ut.cpp | 335 -------- .../CMakeLists.darwin-arm64.txt | 86 --- .../CMakeLists.darwin-x86_64.txt | 87 --- .../CMakeLists.linux-aarch64.txt | 90 --- .../CMakeLists.linux-x86_64.txt | 92 --- .../ut_range_avl_tree/CMakeLists.txt | 19 - .../CMakeLists.windows-x86_64.txt | 80 -- .../tx/datashard/ut_range_avl_tree/ya.make | 38 - ydb/core/tx/datashard/ya.make | 3 - ydb/core/tx/locks/locks.cpp | 22 +- ydb/core/tx/locks/locks.h | 10 +- 17 files changed, 30 insertions(+), 1583 deletions(-) delete mode 100644 ydb/core/tx/datashard/range_avl_tree.cpp delete mode 100644 ydb/core/tx/datashard/range_avl_tree.h delete mode 100644 ydb/core/tx/datashard/range_avl_tree_ut.cpp delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-arm64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-x86_64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-aarch64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-x86_64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.windows-x86_64.txt delete mode 100644 ydb/core/tx/datashard/ut_range_avl_tree/ya.make diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 285ffd11b69d..488ea4b1b962 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -37,6 +37,7 @@ void TTxInit::SetDefaults() { Self->LastWriteId = TWriteId{0}; Self->LastPlannedStep = 0; Self->LastPlannedTxId = 0; + Self->LastCompletedTx = NOlap::TSnapshot::Zero(); Self->OwnerPathId = 0; Self->OwnerPath.clear(); Self->LongTxWrites.clear(); @@ -72,6 +73,14 @@ bool TTxInit::Precharge(TTransactionContext& txc) { ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::OwnerPathId, Self->OwnerPathId); ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::OwnerPath, Self->OwnerPath); + { + ui64 lastCompletedStep = 0; + ui64 lastCompletedTx = 0; + ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::LastCompletedStep, lastCompletedStep); + ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::LastCompletedTxId, lastCompletedTx); + Self->LastCompletedTx = NOlap::TSnapshot(lastCompletedStep, lastCompletedTx); + } + if (!ready) { return false; } diff --git a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp index 42597d49275d..e07290c68140 100644 --- a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp +++ b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp @@ -31,6 +31,12 @@ class TColumnShard::TTxProgressTx : public TTransactionBase { if (!!plannedItem) { ui64 step = plannedItem->PlanStep; ui64 txId = plannedItem->TxId; + LastCompletedTx = NOlap::TSnapshot(step, txId); + if (LastCompletedTx > Self->LastCompletedTx) { + NIceDb::TNiceDb db(txc.DB); + Schema::SaveSpecialValue(db, Schema::EValueIds::LastCompletedStep, LastCompletedTx->GetPlanStep()); + Schema::SaveSpecialValue(db, Schema::EValueIds::LastCompletedTxId, LastCompletedTx->GetTxId()); + } TxOperator = Self->ProgressTxController->GetVerifiedTxOperator(txId); AFL_VERIFY(TxOperator->Progress(*Self, NOlap::TSnapshot(step, txId), txc)); @@ -50,12 +56,16 @@ class TColumnShard::TTxProgressTx : public TTransactionBase { if (TxOperator) { TxOperator->Complete(*Self, ctx); } + if (LastCompletedTx) { + Self->LastCompletedTx = std::max(*LastCompletedTx, Self->LastCompletedTx); + } Self->SetupIndexation(); } private: TTxController::ITransactionOperatior::TPtr TxOperator; const ui32 TabletTxNo; + std::optional LastCompletedTx; }; void TColumnShard::EnqueueProgressTx(const TActorContext& ctx) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 8bd78793ed56..ecd1e8ecc756 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -239,7 +239,7 @@ class TColumnShard } inline TRowVersion LastCompleteTxVersion() const { - return TRowVersion(LastCompletedStep, LastCompletedTxId); + return TRowVersion(LastCompletedTx.GetPlanStep(), LastCompletedTx.GetTxId()); } ui32 Generation() const { return Executor()->Generation(); } @@ -309,9 +309,6 @@ class TColumnShard } } -public: - TTabletCountersBase* TabletCounters; - private: std::unique_ptr ProgressTxController; std::unique_ptr OperationsManager; @@ -391,8 +388,7 @@ class TColumnShard TWriteId LastWriteId = TWriteId{0}; ui64 LastPlannedStep = 0; ui64 LastPlannedTxId = 0; - ui64 LastCompletedStep = 0; - ui64 LastCompletedTxId = 0; + NOlap::TSnapshot LastCompletedTx = NOlap::TSnapshot::Zero(); ui64 LastExportNo = 0; ui64 OwnerPathId = 0; @@ -419,6 +415,7 @@ class TColumnShard TTablesManager TablesManager; std::shared_ptr Tiers; std::unique_ptr TabletCountersPtr; + TTabletCountersBase* TabletCounters; std::unique_ptr PipeClientCache; std::unique_ptr InsertTable; std::shared_ptr SubscribeCounters; diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index f7a1b20b0bb4..0c66980fb3c3 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -64,6 +64,8 @@ struct Schema : NIceDb::Schema { LastExportNumber = 10, OwnerPathId = 11, OwnerPath = 12, + LastCompletedStep = 13, + LastCompletedTxId = 14, }; enum class EInsertTableIds : ui8 { diff --git a/ydb/core/tx/datashard/range_avl_tree.cpp b/ydb/core/tx/datashard/range_avl_tree.cpp deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ydb/core/tx/datashard/range_avl_tree.h b/ydb/core/tx/datashard/range_avl_tree.h deleted file mode 100644 index c568b4b0ed73..000000000000 --- a/ydb/core/tx/datashard/range_avl_tree.h +++ /dev/null @@ -1,721 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -namespace NKikimr { -namespace NDataShard { - - template - class TRangeAvlTree : public TRangeTreeBase { - public: - TRangeAvlTree() = default; - - ~TRangeAvlTree() noexcept { - delete Root; - } - - private: - class TNode : public TIntrusiveListItem { - public: - TNode* Parent = nullptr; - TNode* Left = nullptr; - TNode* Right = nullptr; - i64 Height = 0; - TOwnedCellVec LeftKey; - TOwnedCellVec RightKey; - TOwnedCellVec MaxRightKey; - TValue Value; - EPrefixMode LeftMode; - EPrefixMode RightMode; - EPrefixMode MaxRightMode; - bool MaxRightTrivial; - - ~TNode() noexcept { - if (Left) { - delete Left; - } - if (Right) { - delete Right; - } - } - - TBorder LeftBorder() const noexcept { - return TBorder{ LeftKey, LeftMode }; - } - - TBorder RightBorder() const noexcept { - return TBorder{ RightKey, RightMode }; - } - - TBorder MaxRightBorder() const noexcept { - return TBorder{ MaxRightKey, MaxRightMode }; - } - - TRange ToRange() const noexcept { - return TRange(LeftKey, TBorder::IsInclusive(LeftMode), RightKey, TBorder::IsInclusive(RightMode)); - } - }; - - private: - /** - * Recomputes height for t only - */ - void RecomputeHeight(TNode* t) noexcept { - i64 lheight = t->Left ? t->Left->Height : 0; - i64 rheight = t->Right ? t->Right->Height : 0; - t->Height = Max(lheight, rheight) + 1; - } - - /** - * Recomputes heights for t and all its parents - */ - void RecomputeHeights(TNode* t) noexcept { - while (t) { - i64 lheight = t->Left ? t->Left->Height : 0; - i64 rheight = t->Right ? t->Right->Height : 0; - i64 height = Max(lheight, rheight) + 1; - - if (t->Height == height) { - return; - } - - t->Height = height; - t = t->Parent; - } - } - - /** - * Recomputes MaxRightKey for node t - */ - void RecomputeMaxRightKey(TNode* t) noexcept { - t->MaxRightKey = t->RightKey; - t->MaxRightMode = t->RightMode; - t->MaxRightTrivial = true; - if (t->Left && CompareBorders(t->MaxRightBorder(), t->Left->MaxRightBorder()) < 0) { - t->MaxRightKey = t->Left->MaxRightKey; - t->MaxRightMode = t->Left->MaxRightMode; - t->MaxRightTrivial = false; - } - if (t->Right && CompareBorders(t->MaxRightBorder(), t->Right->MaxRightBorder()) < 0) { - t->MaxRightKey = t->Right->MaxRightKey; - t->MaxRightMode = t->Right->MaxRightMode; - t->MaxRightTrivial = false; - } - } - - /** - * Extends MaxRightKey for node t - * - * Returns true when t->MaxRightKey is modified - */ - bool ExtendMaxRightKey(TNode* t, const TOwnedCellVec& rightKey, EPrefixMode rightMode) noexcept { - if (CompareBorders(t->MaxRightBorder(), TBorder{ rightKey, rightMode }) < 0) { - t->MaxRightKey = rightKey; - t->MaxRightMode = rightMode; - t->MaxRightTrivial = false; - return true; - } else { - return false; - } - } - - /** - * Extends MaxRightKey for node t and all parents - */ - void ExtendMaxRightKeys(TNode* t, const TOwnedCellVec& rightKey, EPrefixMode rightMode) noexcept { - while (t && ExtendMaxRightKey(t, rightKey, rightMode)) { - t = t->Parent; - } - } - - /** - * Extends RightKey for node t when necessary - * - * Updates MaxRightKey for all parents accordingly - */ - void ExtendRightKey(TNode* t, const TBorder& rightKey, TOwnedCellVec ownedRightKey) { - if (CompareBorders(t->RightBorder(), rightKey) < 0) { - t->RightKey = ownedRightKey; - t->RightMode = rightKey.Mode; - if (t->MaxRightTrivial) { - t->MaxRightKey = t->RightKey; - t->MaxRightMode = t->RightMode; - ExtendMaxRightKeys(t->Parent, t->RightKey, t->RightMode); - } else { - int cmp = CompareBorders(t->MaxRightBorder(), t->RightBorder()); - if (cmp <= 0) { - t->MaxRightKey = t->RightKey; - t->MaxRightTrivial = true; - t->MaxRightMode = t->RightMode; - } - if (cmp < 0) { - ExtendMaxRightKeys(t->Parent, t->RightKey, t->RightMode); - } - } - } - } - - TNode* FindFirstUnbalancedGrandParent(TNode* t) noexcept { - if (t == nullptr || t->Parent == nullptr) { - return nullptr; - } - - TNode* gp = t->Parent->Parent; - while (gp) { - i64 lheight = gp->Left ? gp->Left->Height : 0; - i64 rheight = gp->Right ? gp->Right->Height : 0; - i64 balance = lheight - rheight; - - if (balance < -1 || balance > +1) { - return t; - } - - t = t->Parent; - gp = gp->Parent; - } - - return nullptr; - } - - TNode* FindFirstUnbalancedAndRecomputeMaxRightKey(TNode* t) noexcept { - while (t) { - i64 lheight = t->Left ? t->Left->Height : 0; - i64 rheight = t->Right ? t->Right->Height : 0; - i64 balance = lheight - rheight; - - if (balance < -1 || balance > +1) { - // Note: we don't recompute MaxRightKey for unbalanced node - // It's going to be rebalanced and recomputed anyway - return t; - } - - RecomputeMaxRightKey(t); - t = t->Parent; - } - - return nullptr; - } - - /** - * Rebalances node n, which must have an unbalanced grandparent - */ - TNode* Rebalance(TNode* n) noexcept { - TNode* a; - TNode* b; - TNode* c; - TNode* t1; - TNode* t2; - TNode* t3; - TNode* t4; - TNode* keep; - - TNode* p = n->Parent; - TNode* gp = p->Parent; - TNode* ggp = gp->Parent; - - if (gp->Right == p) { - if (p->Right == n) { - // Right Right Case - a = gp; - b = p; - c = n; - t1 = gp->Left; - t2 = p->Left; - t3 = n->Left; - t4 = n->Right; - keep = n; - } else { - // Right Left Case - a = gp; - b = n; - c = p; - t1 = gp->Left; - t2 = n->Left; - t3 = n->Right; - t4 = p->Right; - keep = nullptr; - } - } else { - if (p->Right == n) { - // Left Right Case - a = p; - b = n; - c = gp; - t1 = p->Left; - t2 = n->Left; - t3 = n->Right; - t4 = gp->Right; - keep = nullptr; - } else { - // Left Left Case - a = n; - b = p; - c = gp; - t1 = n->Left; - t2 = n->Right; - t3 = p->Right; - t4 = gp->Right; - keep = n; - } - } - - if (ggp == nullptr) { - Root = b; - } else if (ggp->Left == gp) { - ggp->Left = b; - } else { - ggp->Right = b; - } - b->Parent = ggp; - - b->Left = a; - a->Parent = b; - - b->Right = c; - c->Parent = b; - - a->Left = t1; - if (t1 != nullptr) { - t1->Parent = a; - } - - a->Right = t2; - if (t2 != nullptr) { - t2->Parent = a; - } - - c->Left = t3; - if (t3 != nullptr) { - t3->Parent = c; - } - - c->Right = t4; - if (t4 != nullptr) { - t4->Parent = c; - } - - if (a != keep) { - RecomputeHeight(a); - RecomputeMaxRightKey(a); - } - - if (c != keep) { - RecomputeHeight(c); - RecomputeMaxRightKey(c); - } - - RecomputeHeight(b); - RecomputeMaxRightKey(b); - - RecomputeHeights(ggp); - return ggp; - } - - private: - void DoInsert( - const TBorder& leftKey, - const TBorder& rightKey, - TOwnedCellVec ownedLeftKey, - TOwnedCellVec ownedRightKey, - TValue value) - { - TNode* current = Root; - TNode* parent = nullptr; - bool isLeft = true; - while (current) { - int cmp = CompareBorders(leftKey, current->LeftBorder()); - if (cmp == 0 && value == current->Value) { - ++Stats_.Updates; - ExtendRightKey(current, rightKey, std::move(ownedRightKey)); - return; - } - - if (cmp < 0 || cmp == 0 && value < current->Value) { - isLeft = true; - parent = current; - current = current->Left; - } else { - isLeft = false; - parent = current; - current = current->Right; - } - } - - THolder p(new TNode); - p->Parent = parent; - p->Height = 1; - p->LeftKey = std::move(ownedLeftKey); - p->RightKey = std::move(ownedRightKey); - p->MaxRightKey = p->RightKey; - p->Value = value; - p->LeftMode = leftKey.Mode; - p->RightMode = rightKey.Mode; - p->MaxRightMode = p->RightMode; - p->MaxRightTrivial = true; - - TNode* inserted = p.Get(); - - if (parent) { - if (isLeft) { - parent->Left = p.Release(); - } else { - parent->Right = p.Release(); - } - - RecomputeHeights(parent); - ExtendMaxRightKeys(parent, inserted->RightKey, inserted->RightMode); - - if (TNode* ub = FindFirstUnbalancedGrandParent(inserted)) { - Rebalance(ub); - } - } else { - Root = p.Release(); - } - - Values[value].PushBack(inserted); - ++Size_; - ++Stats_.Inserts; - } - - void DoRemove(TNode* n) noexcept { - TNode* fixfrom; - - if (n->Right && (!n->Left || n->Left->Height <= n->Right->Height)) { - TNode* replacement = n->Right; - while (replacement->Left) { - replacement = replacement->Left; - } - - if (replacement->Parent == n) { - fixfrom = replacement; - } else { - fixfrom = replacement->Parent; - } - - DoRemoveNode(replacement, replacement->Right); - DoReplaceNode(n, replacement); - } else if (n->Left) { - TNode* replacement = n->Left; - while (replacement->Right) { - replacement = replacement->Right; - } - - if (replacement->Parent == n) { - fixfrom = replacement; - } else { - fixfrom = replacement->Parent; - } - - DoRemoveNode(replacement, replacement->Left); - DoReplaceNode(n, replacement); - } else { - fixfrom = n->Parent; - - DoRemoveNode(n, nullptr); - } - - delete n; - --Size_; - ++Stats_.Deletes; - - if (fixfrom == nullptr) { - return; - } - - RecomputeHeights(fixfrom); - - while (TNode* ub = FindFirstUnbalancedAndRecomputeMaxRightKey(fixfrom)) { - i64 lheight = ub->Left ? ub->Left->Height : 0; - i64 rheight = ub->Right ? ub->Right->Height : 0; - - if (rheight >= lheight) { - ub = ub->Right; - lheight = ub->Left ? ub->Left->Height : 0; - rheight = ub->Right ? ub->Right->Height : 0; - } else { - ub = ub->Left; - lheight = ub->Left ? ub->Left->Height : 0; - rheight = ub->Right ? ub->Right->Height : 0; - } - - if (rheight >= lheight) { - ub = ub->Right; - } else { - ub = ub->Left; - } - - fixfrom = Rebalance(ub); - } - } - - void DoRemoveNode(TNode* n, TNode* filler) noexcept { - TNode* parent = n->Parent; - if (parent) { - if (parent->Left == n) { - parent->Left = filler; - } else { - parent->Right = filler; - } - } else { - Root = filler; - } - - if (filler) { - filler->Parent = parent; - } - - n->Left = nullptr; - n->Right = nullptr; - } - - void DoReplaceNode(TNode* n, TNode* replacement) noexcept { - TNode* parent = n->Parent; - TNode* left = n->Left; - TNode* right = n->Right; - - replacement->Left = left; - if (left) { - left->Parent = replacement; - } - - replacement->Right = right; - if (right) { - right->Parent = replacement; - } - - replacement->Parent = parent; - - if (parent) { - if (parent->Left == n) { - parent->Left = replacement; - } else { - parent->Right = replacement; - } - } else { - Root = replacement; - } - - replacement->Height = n->Height; - n->Left = nullptr; - n->Right = nullptr; - } - - private: - /** - * Validates all invariants for subtree t - */ - std::tuple DoValidate(TNode* t) const noexcept { - int cmp; - TNode* leftMost = t; - TNode* rightMost = t; - TBorder maxRightBorder = t->RightBorder(); - bool maxRightTrivial = true; - i64 maxHeight = 1; - - if (auto* l = t->Left) { - Y_ABORT_UNLESS(l->Parent == t, "Left child parent is incorrect"); - cmp = this->CompareBorders(l->LeftBorder(), t->LeftBorder()); - Y_ABORT_UNLESS(cmp < 0 || cmp == 0 && l->Value < t->Value, "Left child must be smaller than t"); - TNode* leftRightMost; - std::tie(leftMost, leftRightMost) = DoValidate(l); - cmp = this->CompareBorders(leftRightMost->LeftBorder(), t->LeftBorder()); - Y_ABORT_UNLESS(cmp < 0 || cmp == 0 && leftRightMost->Value < t->Value, "Left child rightmost node must be smaller than t"); - cmp = this->CompareBorders(maxRightBorder, l->MaxRightBorder()); - if (cmp < 0) { - maxRightBorder = l->MaxRightBorder(); - maxRightTrivial = false; - } - maxHeight = Max(maxHeight, l->Height + 1); - } - - if (auto* r = t->Right) { - Y_ABORT_UNLESS(r->Parent == t, "Right child parent is incorrect"); - cmp = this->CompareBorders(t->LeftBorder(), r->LeftBorder()); - Y_ABORT_UNLESS(cmp < 0 || cmp == 0 && t->Value < r->Value, "Right child must be bigger than t"); - TNode* rightLeftMost; - std::tie(rightLeftMost, rightMost) = DoValidate(r); - cmp = this->CompareBorders(t->LeftBorder(), rightLeftMost->LeftBorder()); - Y_ABORT_UNLESS(cmp < 0 || cmp == 0 && t->Value < rightLeftMost->Value, "Right child leftmost node must be bigger than t"); - cmp = this->CompareBorders(maxRightBorder, r->MaxRightBorder()); - if (cmp < 0) { - maxRightBorder = r->MaxRightBorder(); - maxRightTrivial = false; - } - maxHeight = Max(maxHeight, r->Height + 1); - } - - Y_ABORT_UNLESS(t->Height == maxHeight, "Subtree height is incorrect"); - - cmp = this->CompareBorders(maxRightBorder, t->MaxRightBorder()); - Y_ABORT_UNLESS(cmp == 0, "Subtree must have max right key equal to the calculated max"); - Y_ABORT_UNLESS(maxRightTrivial == t->MaxRightTrivial, - "Subtree must have correct MaxRightTrivial flag (computed=%d, stored=%d)", - int(maxRightTrivial), int(t->MaxRightTrivial)); - - return { leftMost, rightMost }; - } - - public: - /** - * Validates all invariants for the tree, used for tests - */ - void Validate() const noexcept { - if (Root) { - Y_ABORT_UNLESS(Root->Parent == nullptr, "Root must not have a parent"); - DoValidate(Root); - } - } - - public: - void Clear() noexcept { - Values.clear(); - if (Root) { - delete std::exchange(Root, nullptr); - } - Size_ = 0; - } - - /** - * Adds mapping from the given range to the given value - */ - void AddRange(const TRange& range, TValue value) { - AddRange(range.ToOwnedRange(), std::move(value)); - } - - /** - * Adds mapping from the given range to the given value - */ - void AddRange(TOwnedRange range, TValue value) { - Y_ABORT_UNLESS(range.LeftKey.size() <= KeyTypes.size(), "Range left key is too large"); - Y_ABORT_UNLESS(range.RightKey.size() <= KeyTypes.size(), "Range right key is too large"); - - auto leftBorder = TBorder::MakeLeft(range.LeftKey, range.LeftInclusive); - auto rightBorder = TBorder::MakeRight(range.RightKey, range.RightInclusive); - - DoInsert( - leftBorder, - rightBorder, - std::move(range.LeftKey), - std::move(range.RightKey), - value); - } - - /** - * Removes all ranges with the given value - */ - void RemoveRanges(TValue value) { - auto it = Values.find(value); - if (it != Values.end()) { - while (!it->second.Empty()) { - DoRemove(it->second.PopFront()); - } - Values.erase(it); - } - } - - /** - * Calls callback(range, value) for each range in the tree - * - * Order is sorted by (range.Left, value, range.Right) tuples. - */ - template - void EachRange(TCallback&& callback) const { - if (Root) { - DoEachRange(Root, callback); - } - } - - /** - * Calls callback for each range with the given point - */ - template - void EachIntersection(TConstArrayRef point, TCallback&& callback) const { - EachIntersection(TRange(point, true, point, true), callback); - } - - /** - * Calls callback for each range intersecting with the query - */ - template - void EachIntersection(const TRange& range, TCallback&& callback) const { - if (Root) { - DoEachIntersection( - Root, - TBorder::MakeLeft(range.LeftKey, range.LeftInclusive), - TBorder::MakeRight(range.RightKey, range.RightInclusive), - callback); - } - } - - size_t Height() const { - return Root ? Root->Height : 0; - } - - private: - template - void DoEachRange(const TNode* t, TCallback&& callback) const { - if (t->Left) { - DoEachRange(t->Left, callback); - } - { - callback(t->ToRange(), t->Value); - } - if (t->Right) { - DoEachRange(t->Right, callback); - } - } - - template - void DoEachIntersection( - const TNode* t, - const TBorder& leftBorder, - const TBorder& rightBorder, - TCallback&& callback, - bool wentLeft = false) const - { - int cmp; - - if (wentLeft) { - cmp = CompareBorders(t->MaxRightBorder(), leftBorder); - if (cmp < 0) { - // There is no intersection with this whole subtree - return; - } - } - - if (t->Left) { - // Descend into the left subtree - // Note it will terminate on subtrees that have small MaxRightBorder - DoEachIntersection(t->Left, leftBorder, rightBorder, callback, true); - } - - cmp = CompareBorders(rightBorder, t->LeftBorder()); - if (cmp < 0) { - // There is no intersection with this node or the right subtree - return; - } - - // N.B. we avoid comparison with RightKey when it is equal to MaxRightKey. - if ((wentLeft && t->MaxRightTrivial) || - (cmp = CompareBorders(leftBorder, t->RightBorder())) <= 0) - { - callback(t->ToRange(), t->Value); - } - - if (t->Right) { - // Descend into the right subtree - DoEachIntersection(t->Right, leftBorder, rightBorder, callback); - } - } - - private: - TNode* Root = nullptr; - THashMap> Values; - }; - -} // namespace NDataShard -} // namespace NKikimr diff --git a/ydb/core/tx/datashard/range_avl_tree_ut.cpp b/ydb/core/tx/datashard/range_avl_tree_ut.cpp deleted file mode 100644 index 012c980cd6ed..000000000000 --- a/ydb/core/tx/datashard/range_avl_tree_ut.cpp +++ /dev/null @@ -1,335 +0,0 @@ -#include "range_avl_tree.h" - -#include -#include -#include - -namespace NKikimr { -namespace NDataShard { - -namespace { - - TVector CreateSchema(size_t n) { - TVector schema; - auto type = NScheme::TTypeInfo(NScheme::NTypeIds::Uint64); - for (size_t i = 0; i < n; ++i) { - schema.emplace_back(type); - } - return schema; - } - -#if 0 - TVector CreateKey(std::initializer_list keys) { - TVector cells(Reserve(keys.size())); - for (ui64 key : keys) { - cells.emplace_back(TCell::Make(key)); - } - return cells; - } -#endif - - TVector CreateKey(ui64 key) { - TVector cells(Reserve(1)); - cells.emplace_back(TCell::Make(key)); - return cells; - } - - void PrintKey(TStringBuilder& builder, TConstArrayRef key, size_t columns) { - if (columns != 1) { - builder << '{'; - } - for (size_t i = 0; i < columns; ++i) { - if (i > 0) { - builder << ','; - } - if (i < key.size()) { - builder << key[i].AsValue(); - } else { - builder << "any"; - } - } - if (columns != 1) { - builder << '}'; - } - } - - void PrintRange(TStringBuilder& builder, const TRangeTreeBase::TRange& range, ui64 value, size_t columns) { - builder << (range.LeftInclusive ? '[' : '('); - PrintKey(builder, range.LeftKey, columns); - builder << ", "; - PrintKey(builder, range.RightKey, columns); - builder << (range.RightInclusive ? ']' : ')'); - builder << " -> "; - builder << value; - } - - class TRangesToString : public TStringBuilder { - public: - TRangesToString(TStringBuilder& builder, size_t columns) - : Builder(builder) - , Columns(columns) - { } - - void operator()(const TRangeTreeBase::TRange& range, ui64 value) { - if (Index++) { - Builder << ','; - Builder << ' '; - } - PrintRange(Builder, range, value, Columns); - } - - private: - TStringBuilder& Builder; - const size_t Columns; - size_t Index = 0; - }; - - TString AvlTreeToString(const TRangeAvlTree& tree) { - TStringBuilder builder; - tree.EachRange(TRangesToString(builder, tree.KeyColumns())); - return builder; - } - - template - TString IntersectionToString(const TRangeAvlTree& tree, const TNeedle& needle) { - TStringBuilder builder; - tree.EachIntersection(needle, TRangesToString(builder, tree.KeyColumns())); - return builder; - } - - struct TCheckValue { - ui64 Left, Right, Value; - - TCheckValue(ui64 left, ui64 right, ui64 value) - : Left(left) - , Right(right) - , Value(value) - { } - - TString ToString() const { - return TStringBuilder() << *this; - } - - friend inline bool operator==(const TCheckValue& a, const TCheckValue& b) { - return a.Left == b.Left && a.Right == b.Right && a.Value == b.Value; - } - - friend inline IOutputStream& operator<<(IOutputStream& out, const TCheckValue& check) { - out << '['; - out << check.Left; - out << ", "; - out << check.Right; - out << "] -> "; - out << check.Value; - return out; - } - }; - -} // namespace - -Y_UNIT_TEST_SUITE(TRangeAvlTree) { - - Y_UNIT_TEST(Simple) { - using TRange = TRangeTreeBase::TRange; - TRangeAvlTree tree; - tree.SetKeyTypes(CreateSchema(1)); - - tree.AddRange(TRange(CreateKey(1), true, CreateKey(10), true), 42); - tree.AddRange(TRange(CreateKey(2), true, CreateKey(20), true), 43); - tree.AddRange(TRange(CreateKey(3), true, CreateKey(30), true), 44); - tree.Validate(); - UNIT_ASSERT_VALUES_EQUAL(AvlTreeToString(tree), "[1, 10] -> 42, [2, 20] -> 43, [3, 30] -> 44"); - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), 3u); - - tree.AddRange(TRange(CreateKey(2), true, CreateKey(40), true), 43); - tree.Validate(); - UNIT_ASSERT_VALUES_EQUAL(AvlTreeToString(tree), "[1, 10] -> 42, [2, 40] -> 43, [3, 30] -> 44"); - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), 3u); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(1)), - "[1, 10] -> 42"); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(2)), - "[1, 10] -> 42, [2, 40] -> 43"); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(3)), - "[1, 10] -> 42, [2, 40] -> 43, [3, 30] -> 44"); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(15)), - "[2, 40] -> 43, [3, 30] -> 44"); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(35)), - "[2, 40] -> 43"); - - UNIT_ASSERT_VALUES_EQUAL( - IntersectionToString(tree, CreateKey(45)), - ""); - - tree.RemoveRanges(43); - tree.Validate(); - UNIT_ASSERT_VALUES_EQUAL(AvlTreeToString(tree), "[1, 10] -> 42, [3, 30] -> 44"); - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), 2u); - - tree.RemoveRanges(42); - tree.Validate(); - UNIT_ASSERT_VALUES_EQUAL(AvlTreeToString(tree), "[3, 30] -> 44"); - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), 1u); - - tree.RemoveRanges(44); - tree.Validate(); - UNIT_ASSERT_VALUES_EQUAL(AvlTreeToString(tree), ""); - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), 0u); - } - - Y_UNIT_TEST(Sequential) { - using TRange = TRangeTreeBase::TRange; - TRangeAvlTree tree; - tree.SetKeyTypes(CreateSchema(1)); - - const size_t nRanges = 1000000; - for (size_t i = 0; i < nRanges; ++i) { - ui64 left = i + 1; - ui64 right = i + 1; - ui64 value = i + 1; - tree.AddRange(TRange(CreateKey(left), true, CreateKey(right), true), value); - } - tree.Validate(); - - auto buildStats = tree.Stats(); - Cerr << "NOTE: building tree of size " << tree.Size() - << " got height " << tree.Height() << " and needed " - << (buildStats.Inserts + buildStats.Updates + buildStats.Deletes) << " ops (" - << buildStats.Inserts << " inserts " - << buildStats.Updates << " updates " - << buildStats.Deletes << " deletes) and " - << buildStats.Comparisons << " comparisons (" - << double(buildStats.Comparisons) / double(buildStats.Inserts + buildStats.Updates + buildStats.Deletes) - << " per op)" - << Endl; - } - - Y_UNIT_TEST(Random) { - using TRange = TRangeTreeBase::TRange; - TRangeAvlTree tree; - tree.SetKeyTypes(CreateSchema(1)); - - using TCheckMap = TMap, ui64>; - TCheckMap map; // (left, value) -> right - using TCheckValues = THashMap>; - TCheckValues values; - - const ui64 nValues = 20; -#if 1 - const size_t nRanges = 10000; - const ui64 totalRangeSize = 10000; - const ui64 singleRangeMinSize = 100; - const ui64 singleRangeMaxSize = 5000; -#else - const size_t nRanges = 1000000; - const ui64 totalRangeSize = 1000000; - const ui64 singleRangeMinSize = 1; - const ui64 singleRangeMaxSize = 1; -#endif - - // Insert a bunch of values - for (size_t i = 0; i < nRanges; ++i) { - ui64 left = 1 + (RandomNumber() % totalRangeSize); - ui64 size = singleRangeMinSize + (RandomNumber() % (singleRangeMaxSize - singleRangeMinSize + 1)); - ui64 right = Min(left + size - 1, ui64(totalRangeSize)); - ui64 value = 1 + (RandomNumber() % nValues); - tree.AddRange(TRange(CreateKey(left), true, CreateKey(right), true), value); - // Add it to the classical map too - auto key = std::make_pair(left, value); - auto it = map.find(key); - if (it == map.end()) { - it = map.emplace(key, right).first; - values[value].emplace_back(it); - } else { - it->second = Max(it->second, right); - } - } - - // Remove some values with 10% probability - for (ui64 value = 1; value <= nValues; ++value) { - ui64 prio = RandomNumber() % 10; - if (prio == 0) { - tree.RemoveRanges(value); - // Remove it from the classical map too - for (auto it : values[value]) { - map.erase(it); - } - values.erase(value); - } - } - - auto buildStats = tree.Stats(); - Cerr << "NOTE: building tree of size " << tree.Size() - << " got height " << tree.Height() << " and needed " - << (buildStats.Inserts + buildStats.Updates + buildStats.Deletes) << " ops (" - << buildStats.Inserts << " inserts " - << buildStats.Updates << " updates " - << buildStats.Deletes << " deletes) and " - << buildStats.Comparisons << " comparisons (" - << double(buildStats.Comparisons) / double(buildStats.Inserts + buildStats.Updates + buildStats.Deletes) - << " per op)" - << Endl; - - // The resulting tree must be valid - tree.Validate(); - - // The resulting tree and shadow map must have the same size - UNIT_ASSERT_VALUES_EQUAL(tree.Size(), map.size()); - - auto checkIt = map.begin(); - tree.EachRange([&](const TRange& range, ui64 value) { - TCheckValue found{ range.LeftKey[0].AsValue(), range.RightKey[0].AsValue(), value }; - UNIT_ASSERT_C(checkIt != map.end(), "AvlTree has more values than the map, e.g.: " << found); - TCheckValue expected{ checkIt->first.first, checkIt->second, checkIt->first.second }; - UNIT_ASSERT_VALUES_EQUAL(found, expected); - ++checkIt; - }); - UNIT_ASSERT_C(checkIt == map.end(), "Map has more values than the tree"); - - // Let's find some intersections and verify them with brute force - for (size_t i = 0; i < 10; ++i) { - ui64 point = 1 + (RandomNumber() % totalRangeSize); - Cerr << "Checking point " << point << Endl; - auto checkIt = map.begin(); - tree.ResetStats(); - size_t foundCount = 0; - tree.EachIntersection(CreateKey(point), [&](const TRange& range, ui64 value) { - TCheckValue found{ range.LeftKey[0].AsValue(), range.RightKey[0].AsValue(), value }; - // Skip all map values that don't intersect with point - while (checkIt != map.end() && !(checkIt->first.first <= point && point <= checkIt->second)) { - ++checkIt; - } - UNIT_ASSERT_C(checkIt != map.end(), "AvlTree returned value that was not found in the map, e.g." << found); - TCheckValue expected{ checkIt->first.first, checkIt->second, checkIt->first.second }; - UNIT_ASSERT_VALUES_EQUAL_C(found, expected, "AvlTree returned a value that does not match with the map"); - ++checkIt; - ++foundCount; - }); - auto foundStats = tree.Stats(); - // Check if there is any other matching value - while (checkIt != map.end() && !(checkIt->first.first <= point && point <= checkIt->second)) { - ++checkIt; - } - UNIT_ASSERT_C(checkIt == map.end(), "Map has a value that was not returned from the tree, e.g." - << TCheckValue(checkIt->first.first, checkIt->second, checkIt->first.second)); - Cerr << "... found " << foundCount << " ranges, needed " - << foundStats.Comparisons << " comparisons (" - << double(foundStats.Comparisons) / double(Max(foundCount, ui64(1))) - << " per range)" - << Endl; - } - } - -} - -} // namespace NDataShard -} // namespace NKikimr diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-arm64.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-arm64.txt deleted file mode 100644 index e7d3955f53f9..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-arm64.txt +++ /dev/null @@ -1,86 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_avl_tree) -target_compile_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC - contrib-libs-cxxsupp - yutil - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_avl_tree_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_avl_tree - TEST_TARGET - ydb-core-tx-datashard-ut_range_avl_tree - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_avl_tree - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_avl_tree) diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index 8bc223982387..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,87 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_avl_tree) -target_compile_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_avl_tree_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_avl_tree - TEST_TARGET - ydb-core-tx-datashard-ut_range_avl_tree - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_avl_tree - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_avl_tree) diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-aarch64.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-aarch64.txt deleted file mode 100644 index 9cabb66c1eee..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,90 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_avl_tree) -target_compile_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_avl_tree_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_avl_tree - TEST_TARGET - ydb-core-tx-datashard-ut_range_avl_tree - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_avl_tree - cpp-malloc-jemalloc -) -vcs_info(ydb-core-tx-datashard-ut_range_avl_tree) diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-x86_64.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-x86_64.txt deleted file mode 100644 index 091e31789905..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,92 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_avl_tree) -target_compile_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_link_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_avl_tree_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_avl_tree - TEST_TARGET - ydb-core-tx-datashard-ut_range_avl_tree - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_avl_tree - cpp-malloc-tcmalloc - libs-tcmalloc-no_percpu_cache -) -vcs_info(ydb-core-tx-datashard-ut_range_avl_tree) diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.txt deleted file mode 100644 index d863ebd18067..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - include(CMakeLists.darwin-arm64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -endif() diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.windows-x86_64.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 6329299a68d7..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,80 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-core-tx-datashard-ut_range_avl_tree) -target_compile_options(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard -) -target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC - contrib-libs-cxxsupp - yutil - library-cpp-cpuid_check - cpp-testing-unittest_main - core-tx-datashard - tx-datashard-ut_common - library-cpp-getopt - cpp-regex-pcre - library-cpp-svnversion - kqp-ut-common - core-testlib-default - ydb-core-tx - udf-service-exception_policy - public-lib-yson_value - cpp-client-ydb_result -) -target_sources(ydb-core-tx-datashard-ut_range_avl_tree PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/range_avl_tree_ut.cpp -) -set_property( - TARGET - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - SPLIT_FACTOR - 1 -) -add_yunittest( - NAME - ydb-core-tx-datashard-ut_range_avl_tree - TEST_TARGET - ydb-core-tx-datashard-ut_range_avl_tree - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-core-tx-datashard-ut_range_avl_tree - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-core-tx-datashard-ut_range_avl_tree - system_allocator -) -vcs_info(ydb-core-tx-datashard-ut_range_avl_tree) diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/ya.make b/ydb/core/tx/datashard/ut_range_avl_tree/ya.make deleted file mode 100644 index 42434bd75a29..000000000000 --- a/ydb/core/tx/datashard/ut_range_avl_tree/ya.make +++ /dev/null @@ -1,38 +0,0 @@ -UNITTEST_FOR(ydb/core/tx/datashard) - -FORK_SUBTESTS() - -SPLIT_FACTOR(1) - -IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) - REQUIREMENTS(ram:16) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( - ydb/core/tx/datashard/ut_common - library/cpp/getopt - library/cpp/regex/pcre - library/cpp/svnversion - ydb/core/kqp/ut/common - ydb/core/testlib/default - ydb/core/tx - ydb/library/yql/public/udf/service/exception_policy - ydb/public/lib/yson_value - ydb/public/sdk/cpp/client/ydb_result -) - -YQL_LAST_ABI_VERSION() - -SRCS( - range_avl_tree_ut.cpp -) - -REQUIREMENTS(ram:32) - -END() diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 12f02b8ed5da..d89f6fa8e5fe 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -181,7 +181,6 @@ SRCS( receive_snapshot_unit.cpp remove_lock_change_records.cpp remove_locks.cpp - range_avl_tree.cpp range_ops.cpp read_iterator.h restore_unit.cpp @@ -292,9 +291,7 @@ RECURSE_FOR_TESTS( ut_minikql ut_minstep ut_order - ut_range_avl_tree ut_range_ops - ut_range_treap ut_read_iterator ut_read_table ut_reassign diff --git a/ydb/core/tx/locks/locks.cpp b/ydb/core/tx/locks/locks.cpp index d9c68e29f64e..1a614f45e491 100644 --- a/ydb/core/tx/locks/locks.cpp +++ b/ydb/core/tx/locks/locks.cpp @@ -567,10 +567,8 @@ void TLockLocker::RemoveOneLock(ui64 lockTxId, ILocksDb* db) { TLockInfo::TPtr txLock = it->second; TDuration lifetime = TAppData::TimeProvider->Now() - txLock->GetCreationTime(); - if (Self->TabletCounters) { - Self->IncCounter(COUNTER_LOCKS_LIFETIME, lifetime); - Self->IncCounter(COUNTER_LOCKS_REMOVED); - } + Self->IncCounter(COUNTER_LOCKS_LIFETIME, lifetime); + Self->IncCounter(COUNTER_LOCKS_REMOVED); ExpireQueue.Remove(txLock.Get()); if (txLock->InBrokenLocks) { @@ -663,9 +661,7 @@ void TLockLocker::ScheduleRemoveBrokenRanges(ui64 lockId, const TRowVersion& at) CleanupPending.push_back(lockId); } - if (Self->TabletCounters) { - Self->IncCounter(COUNTER_LOCKS_BROKEN); - } + Self->IncCounter(COUNTER_LOCKS_BROKEN); } void TLockLocker::RemoveSubscribedLock(ui64 lockId, ILocksDb* db) { @@ -744,7 +740,7 @@ TVector TSysLocks::ApplyLocks() { ++erases; } - if (erases > 0 && Self->TabletCounters) { + if (erases > 0) { Self->IncCounter(COUNTER_LOCKS_ERASED, erases); } @@ -777,9 +773,7 @@ TVector TSysLocks::ApplyLocks() { } else { if (shardLock) { Locker.AddShardLock(lock, Update->ReadTables); - if (Self->TabletCounters) { - Self->IncCounter(COUNTER_LOCKS_WHOLE_SHARD); - } + Self->IncCounter(COUNTER_LOCKS_WHOLE_SHARD); } else { for (const auto& key : Update->PointLocks) { Locker.AddPointLock(lock, key); @@ -828,17 +822,11 @@ TVector TSysLocks::ApplyLocks() { } void TSysLocks::UpdateCounters() { - if (!Self->TabletCounters) - return; - Self->IncCounter(COUNTER_LOCKS_ACTIVE_PER_SHARD, LocksCount()); Self->IncCounter(COUNTER_LOCKS_BROKEN_PER_SHARD, BrokenLocksCount()); } void TSysLocks::UpdateCounters(ui64 counter) { - if (!Self->TabletCounters) - return; - UpdateCounters(); if (TLock::IsError(counter)) { diff --git a/ydb/core/tx/locks/locks.h b/ydb/core/tx/locks/locks.h index 05b788594491..31a70f11626e 100644 --- a/ydb/core/tx/locks/locks.h +++ b/ydb/core/tx/locks/locks.h @@ -73,11 +73,6 @@ class ILocksDb { class TLocksDataShard { public: - TLocksDataShard(TTabletCountersBase* const &tabletCounters) - : TabletCounters(tabletCounters) - { - } - virtual ~TLocksDataShard() = default; virtual void IncCounter(ECumulativeCounters counter, @@ -91,8 +86,6 @@ class TLocksDataShard { virtual bool IsUserTable(const TTableId& tableId) const = 0; virtual ui32 Generation() const = 0; virtual TRowVersion LastCompleteTxVersion() const = 0; - - TTabletCountersBase* const &TabletCounters; }; template @@ -100,8 +93,7 @@ class TLocksDataShardAdapter : public TLocksDataShard { public: TLocksDataShardAdapter(const T *self) - : TLocksDataShard(self->TabletCounters) - , Self(self) + : Self(self) { }