From e65827232a394af653c2fc372219e7dea123a6e3 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 19 Apr 2022 10:17:29 +0800 Subject: [PATCH 01/38] Add v3 mix mode Signed-off-by: jiaqizho --- dbms/src/Interpreters/Context.cpp | 47 ++---- dbms/src/Interpreters/Context.h | 8 +- dbms/src/Server/DTTool/DTToolBench.cpp | 2 +- dbms/src/Server/Server.cpp | 6 +- dbms/src/Server/tests/gtest_dttool.cpp | 2 +- dbms/src/Server/tests/gtest_server_config.cpp | 2 +- .../DeltaMerge/ColumnFile/ColumnFileBig.cpp | 2 +- .../Delta/ColumnFilePersistedSet.cpp | 2 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 17 +- dbms/src/Storages/DeltaMerge/Segment.cpp | 2 +- .../Storages/DeltaMerge/StableValueSpace.cpp | 4 +- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 153 ++++++++++++------ dbms/src/Storages/DeltaMerge/StoragePool.h | 84 +++++++--- .../tests/gtest_dm_delta_value_space.cpp | 2 +- .../DeltaMerge/tests/gtest_dm_file.cpp | 4 +- .../DeltaMerge/tests/gtest_dm_segment.cpp | 2 +- .../tests/gtest_dm_segment_common_handle.cpp | 2 +- dbms/src/Storages/Page/PageStorage.h | 22 +++ .../Storages/Transaction/RegionPersister.cpp | 38 +++-- dbms/src/TestUtils/TiFlashTestEnv.cpp | 5 +- 20 files changed, 254 insertions(+), 152 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index ce4ecc692c0..da2717f78b4 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -161,7 +161,7 @@ struct ContextShared PathCapacityMetricsPtr path_capacity_ptr; /// Path capacity metrics FileProviderPtr file_provider; /// File provider. IORateLimiter io_rate_limiter; - DM::GlobalStoragePoolPtr global_storage_pool; + DM::StoragePoolRunMode storage_run_mode; /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. class SessionKeyHash @@ -1547,21 +1547,8 @@ ReadLimiterPtr Context::getReadLimiter() const return getIORateLimiter().getReadLimiter(); } -static bool isUsingPageStorageV3(const PathPool & path_pool, bool enable_ps_v3) +static bool isPageStorageV2Existed(const PathPool & path_pool) { - // Check whether v3 is already enabled - for (const auto & path : path_pool.listGlobalPagePaths()) - { - if (PS::V3::PageStorageImpl::isManifestsFileExists(path)) - { - return true; - } - } - - // Check whether v3 on new node is enabled in the config, if not, no need to check anymore - if (!enable_ps_v3) - return false; - // Check whether there are any files in kvstore path, if exists, then this is not a new node. // If it's a new node, then we enable v3. Otherwise, we use v2. for (const auto & path : path_pool.listKVStorePaths()) @@ -1580,38 +1567,22 @@ static bool isUsingPageStorageV3(const PathPool & path_pool, bool enable_ps_v3) return true; } -bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool, bool enable_ps_v3) +void Context::initializeStoragePoolMode(const PathPool & path_pool, bool enable_ps_v3) { auto lock = getLock(); - if (isUsingPageStorageV3(path_pool, enable_ps_v3)) - { - try - { - // create manifests file before initialize GlobalStoragePool - for (const auto & path : path_pool.listGlobalPagePaths()) - PS::V3::PageStorageImpl::createManifestsFileIfNeed(path); - shared->global_storage_pool = std::make_shared(path_pool, *this, settings); - shared->global_storage_pool->restore(); - return true; - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - throw; - } - } - else + if (!enable_ps_v3) { - shared->global_storage_pool = nullptr; - return false; + shared->storage_run_mode = DM::StoragePoolRunMode::ONLY_V2; } + + shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? DM::StoragePoolRunMode::MIX_MODE : DM::StoragePoolRunMode::ONLY_V3; } -DM::GlobalStoragePoolPtr Context::getGlobalStoragePool() const +DM::StoragePoolRunMode Context::getStoragePoolRunMode() const { auto lock = getLock(); - return shared->global_storage_pool; + return shared->storage_run_mode; } UInt16 Context::getTCPPort() const diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 9e8a1b248ba..d81f1ee9370 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -103,8 +103,7 @@ namespace DM { class MinMaxIndexCache; class DeltaIndexManager; -class GlobalStoragePool; -using GlobalStoragePoolPtr = std::shared_ptr; +enum class StoragePoolRunMode : UInt8; } // namespace DM /// (database name, table name) @@ -405,9 +404,8 @@ class Context ReadLimiterPtr getReadLimiter() const; IORateLimiter & getIORateLimiter() const; - bool initializeGlobalStoragePoolIfNeed(const PathPool & path_pool, bool enable_ps_v3); - - DM::GlobalStoragePoolPtr getGlobalStoragePool() const; + void initializeStoragePoolMode(const PathPool & path_pool, bool enable_ps_v3); + DM::StoragePoolRunMode getStoragePoolRunMode() const; /// Call after initialization before using system logs. Call for global context. void initializeSystemLogs(); diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index 491ac710721..82053e93ef6 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -336,7 +336,7 @@ int benchEntry(const std::vector & opts) auto settings = DB::Settings(); auto db_context = env.getContext(); auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique("test.t1", /*table_id*/ 1, *path_pool, *db_context, db_context->getSettingsRef()); + auto storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 4762935cd6f..8187b7ef487 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1117,8 +1117,10 @@ int Server::main(const std::vector & /*args*/) // 2. initialize KVStore service // 1) because we need to check whether this is the first startup of this node, and we judge it based on whether there are any files in kvstore directory // 2) KVStore service also choose its data format based on whether the GlobalStoragePool is initialized - if (global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool(), storage_config.enable_ps_v3)) - LOG_FMT_INFO(log, "PageStorage V3 enabled."); + + // TODO + // if (global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool(), storage_config.enable_ps_v3)) + // LOG_FMT_INFO(log, "PageStorage V3 enabled."); // Use pd address to define which default_database we use by default. // For mock test, we use "default". For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index 6b0d6e3d5c2..b79edb69c33 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -72,7 +72,7 @@ struct DTToolTest : public DB::base::TiFlashStorageTestBasic properties.push_back(property); } auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique("test.t1", /*table_id*/ 1, *path_pool, *db_context, db_context->getSettingsRef()); + auto storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index 01f2424d939..46aae7e4740 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -363,7 +363,7 @@ dt_page_gc_low_write_prob = 0.2 auto & global_ctx = TiFlashTestEnv::getGlobalContext(); std::unique_ptr path_pool = std::make_unique(global_ctx.getPathPool().withTable("test", "t1", false)); - std::unique_ptr storage_pool = std::make_unique("test.t1", /*table_id*/ 100, *path_pool, global_ctx, global_ctx.getSettingsRef()); + std::unique_ptr storage_pool = std::make_unique(DM::StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, global_ctx, "test.t1"); auto verify_storage_pool_reload_config = [&global_ctx](std::unique_ptr & storage_pool) { DB::Settings & settings = global_ctx.getSettingsRef(); diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.cpp index 9fe72839628..306f9470f5d 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.cpp @@ -72,7 +72,7 @@ ColumnFilePersistedPtr ColumnFileBig::deserializeMetadata(DMContext & context, / readIntBinary(valid_rows, buf); readIntBinary(valid_bytes, buf); - auto file_id = context.storage_pool.dataReader().getNormalPageId(file_ref_id); + auto file_id = context.storage_pool.dataReader()->getNormalPageId(file_ref_id); auto file_parent_path = context.path_pool.getStableDiskDelegator().getDTFilePath(file_id); auto dmfile = DMFile::restore(context.db_context.getFileProvider(), file_id, file_ref_id, file_parent_path, DMFile::ReadMetaMode::all()); diff --git a/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp b/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp index b96acdad424..289caf5816c 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp @@ -107,7 +107,7 @@ ColumnFilePersistedSet::ColumnFilePersistedSet(PageId metadata_id_, const Column ColumnFilePersistedSetPtr ColumnFilePersistedSet::restore(DMContext & context, const RowKeyRange & segment_range, PageId id) { - Page page = context.storage_pool.metaReader().read(id); + Page page = context.storage_pool.metaReader()->read(id); ReadBufferFromMemory buf(page.data.begin(), page.data.size()); auto column_files = deserializeSavedColumnFiles(context, segment_range, buf); return std::make_shared(id, column_files); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 80b1d81f817..272c09512bc 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -214,15 +214,20 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; - if (auto global_storage_pool = global_context.getGlobalStoragePool(); global_storage_pool) - { - storage_pool = std::make_shared(ns_id, *global_storage_pool, global_context); - } - else + + const auto & storage_pool_run_mode = global_context.getStoragePoolRunMode(); + if (storage_pool_run_mode == StoragePoolRunMode::ONLY_V3 || storage_pool_run_mode == StoragePoolRunMode::MIX_MODE) { - storage_pool = std::make_shared(db_name_ + "." + table_name_, ns_id, path_pool, global_context, db_context.getSettingsRef()); + GlobalStoragePool::init(global_context.getPathPool(), global_context, db_context.getSettingsRef()); } + storage_pool = std::make_shared(storage_pool_run_mode, + ns_id, + GlobalStoragePool::getInstance(), + path_pool, + global_context, + db_name_ + "." + table_name_); + // Restore existing dm files and set capacity for path_pool. // Should be done before any background task setup. restoreStableFiles(); diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index ac192ff6082..156591b29ff 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -236,7 +236,7 @@ SegmentPtr Segment::newSegment( SegmentPtr Segment::restoreSegment(DMContext & context, PageId segment_id) { - Page page = context.storage_pool.metaReader().read(segment_id); // not limit restore + Page page = context.storage_pool.metaReader()->read(segment_id); // not limit restore ReadBufferFromMemory buf(page.data.begin(), page.data.size()); SegmentFormat::Version version; diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index 648ffd4f084..752a112b565 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -90,7 +90,7 @@ StableValueSpacePtr StableValueSpace::restore(DMContext & context, PageId id) { auto stable = std::make_shared(id); - Page page = context.storage_pool.metaReader().read(id); // not limit restore + Page page = context.storage_pool.metaReader()->read(id); // not limit restore ReadBufferFromMemory buf(page.data.begin(), page.data.size()); UInt64 version, valid_rows, valid_bytes, size; readIntBinary(version, buf); @@ -105,7 +105,7 @@ StableValueSpacePtr StableValueSpace::restore(DMContext & context, PageId id) { readIntBinary(ref_id, buf); - auto file_id = context.storage_pool.dataReader().getNormalPageId(ref_id); + auto file_id = context.storage_pool.dataReader()->getNormalPageId(ref_id); auto file_parent_path = context.path_pool.getStableDiskDelegator().getDTFilePath(file_id); auto dmfile = DMFile::restore(context.db_context.getFileProvider(), file_id, ref_id, file_parent_path, DMFile::ReadMetaMode::all()); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 8cc7dd93f48..b1af99fb9a4 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace DB @@ -87,6 +86,7 @@ static bool doStoragePoolGC(const Context & global_context, const Settings & set return done_anything; } +std::shared_ptr GlobalStoragePool::global_storage_pool = nullptr; GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings) : // The iops and bandwidth in log_storage are relatively high, use multi-disks if possible log_storage(PageStorage::create("__global__.log", @@ -148,59 +148,110 @@ bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_per return doStoragePoolGC(global_context, settings, *this); } - -StoragePool::StoragePool(const String & name, NamespaceId ns_id_, StoragePathPool & path_pool, Context & global_ctx, const Settings & settings) - : owned_storage(true) +StoragePool::StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & storage_path_pool, Context & global_ctx, const String & name) + : run_mode(mode) , ns_id(ns_id_) - , - // The iops and bandwidth in log_storage are relatively high, use multi-disks if possible - log_storage(PageStorage::create(name + ".log", - path_pool.getPSDiskDelegatorMulti("log"), - extractConfig(settings, StorageType::Log), - global_ctx.getFileProvider())) - , - // The iops in data_storage is low, only use the first disk for storing data - data_storage(PageStorage::create(name + ".data", - path_pool.getPSDiskDelegatorSingle("data"), - extractConfig(settings, StorageType::Data), - global_ctx.getFileProvider())) - , - // The iops in meta_storage is relatively high, use multi-disks if possible - meta_storage(PageStorage::create(name + ".meta", - path_pool.getPSDiskDelegatorMulti("meta"), - extractConfig(settings, StorageType::Meta), - global_ctx.getFileProvider())) - , log_storage_reader(ns_id, log_storage, nullptr) - , data_storage_reader(ns_id, data_storage, nullptr) - , meta_storage_reader(ns_id, meta_storage, nullptr) , global_context(global_ctx) -{} +{ + switch (run_mode) + { + case StoragePoolRunMode::ONLY_V2: + { + log_storage_v2 = PageStorage::create(name + ".log", + storage_path_pool.getPSDiskDelegatorMulti("log"), + extractConfig(global_context.getSettingsRef(), StorageType::Log), + global_context.getFileProvider()); + data_storage_v2 = PageStorage::create(name + ".data", + storage_path_pool.getPSDiskDelegatorSingle("data"), + extractConfig(global_context.getSettingsRef(), StorageType::Data), + global_ctx.getFileProvider()); + meta_storage_v2 = PageStorage::create(name + ".meta", + storage_path_pool.getPSDiskDelegatorMulti("meta"), + extractConfig(global_context.getSettingsRef(), StorageType::Meta), + global_ctx.getFileProvider()); + log_storage_reader = std::make_shared(ns_id, log_storage_v2, nullptr); + data_storage_reader = std::make_shared(ns_id, data_storage_v2, nullptr); + meta_storage_reader = std::make_shared(ns_id, meta_storage_v2, nullptr); + break; + } + case StoragePoolRunMode::ONLY_V3: + { + assert(global_storage_pool != nullptr); + log_storage_v3 = global_storage_pool->log(); + data_storage_v3 = global_storage_pool->data(); + meta_storage_v3 = global_storage_pool->meta(); -StoragePool::StoragePool(NamespaceId ns_id_, const GlobalStoragePool & global_storage_pool, Context & global_ctx) - : owned_storage(false) - , ns_id(ns_id_) - , log_storage(global_storage_pool.log()) - , data_storage(global_storage_pool.data()) - , meta_storage(global_storage_pool.meta()) - , log_storage_reader(ns_id, log_storage, nullptr) - , data_storage_reader(ns_id, data_storage, nullptr) - , meta_storage_reader(ns_id, meta_storage, nullptr) - , global_context(global_ctx) -{} + log_storage_reader = std::make_shared(ns_id, log_storage_v3, nullptr); + data_storage_reader = std::make_shared(ns_id, data_storage_v3, nullptr); + meta_storage_reader = std::make_shared(ns_id, meta_storage_v3, nullptr); + + break; + } + case StoragePoolRunMode::MIX_MODE: + { + assert(global_storage_pool != nullptr); + log_storage_v3 = global_storage_pool->log(); + data_storage_v3 = global_storage_pool->data(); + meta_storage_v3 = global_storage_pool->meta(); + + log_storage_v2 = PageStorage::create(name + ".log", + storage_path_pool.getPSDiskDelegatorMulti("log"), + extractConfig(global_context.getSettingsRef(), StorageType::Log), + global_context.getFileProvider()); + data_storage_v2 = PageStorage::create(name + ".data", + storage_path_pool.getPSDiskDelegatorSingle("data"), + extractConfig(global_context.getSettingsRef(), StorageType::Data), + global_ctx.getFileProvider()); + data_storage_v2 = PageStorage::create(name + ".meta", + storage_path_pool.getPSDiskDelegatorMulti("meta"), + extractConfig(global_context.getSettingsRef(), StorageType::Meta), + global_ctx.getFileProvider()); + + // log_storage_reader = std::make_shared(ns_id, log_storage, nullptr); + // data_storage_reader = std::make_shared(ns_id, data_storage, nullptr); + // meta_storage_reader = std::make_shared(ns_id, meta_storage, nullptr); + + break; + } + default: + throw Exception(fmt::format("Unknown StoragePoolRunMode {}", static_cast(run_mode))); + } +} void StoragePool::restore() { // If the storage instances is not global, we need to initialize it by ourselves and add a gc task. - if (owned_storage) + if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) { - log_storage->restore(); - data_storage->restore(); - meta_storage->restore(); + log_storage_v2->restore(); + data_storage_v2->restore(); + meta_storage_v2->restore(); } - max_log_page_id = log_storage->getMaxId(ns_id); - max_data_page_id = data_storage->getMaxId(ns_id); - max_meta_page_id = meta_storage->getMaxId(ns_id); + switch (run_mode) + { + case StoragePoolRunMode::ONLY_V2: + { + max_log_page_id = log_storage_v2->getMaxId(ns_id); + max_data_page_id = data_storage_v2->getMaxId(ns_id); + max_meta_page_id = meta_storage_v2->getMaxId(ns_id); + break; + } + case StoragePoolRunMode::ONLY_V3: + { + max_log_page_id = log_storage_v3->getMaxId(ns_id); + max_data_page_id = data_storage_v3->getMaxId(ns_id); + max_meta_page_id = meta_storage_v3->getMaxId(ns_id); + break; + } + case StoragePoolRunMode::MIX_MODE: + { + max_log_page_id = std::max(log_storage_v2->getMaxId(ns_id), log_storage_v3->getMaxId(ns_id)); + max_data_page_id = std::max(data_storage_v2->getMaxId(ns_id), data_storage_v3->getMaxId(ns_id)); + max_meta_page_id = std::max(meta_storage_v2->getMaxId(ns_id), meta_storage_v3->getMaxId(ns_id)); + break; + } + } } StoragePool::~StoragePool() @@ -210,13 +261,15 @@ StoragePool::~StoragePool() void StoragePool::enableGC() { - if (owned_storage) + if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) + { gc_handle = global_context.getBackgroundPool().addTask([this] { return this->gc(global_context.getSettingsRef()); }); + } } bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { - if (!owned_storage) + if (run_mode == StoragePoolRunMode::ONLY_V3) return false; { @@ -246,11 +299,11 @@ void StoragePool::drop() { shutdown(); - if (owned_storage) + if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) { - meta_storage->drop(); - data_storage->drop(); - log_storage->drop(); + meta_storage_v2->drop(); + data_storage_v2->drop(); + log_storage_v2->drop(); } } diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 859be76dbe5..e986a6b7b59 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -31,11 +32,16 @@ namespace DM { class StoragePool; using StoragePoolPtr = std::shared_ptr; -class GlobalStoragePool; -using GlobalStoragePoolPtr = std::shared_ptr; static const std::chrono::seconds DELTA_MERGE_GC_PERIOD(60); +enum class StoragePoolRunMode : UInt8 +{ + ONLY_V2 = 1, + ONLY_V3 = 2, + MIX_MODE = 3, +}; + class GlobalStoragePool : private boost::noncopyable { public: @@ -43,12 +49,37 @@ class GlobalStoragePool : private boost::noncopyable using Timepoint = Clock::time_point; using Seconds = std::chrono::seconds; - GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); + // not thread safe + static void init(const PathPool & path_pool, Context & global_ctx, const Settings & settings) + { + if (global_storage_pool != nullptr) + { + return; + } + + try + { + global_storage_pool = std::make_shared(path_pool, global_ctx, settings); + global_storage_pool->restore(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + throw; + } + } - void restore(); + static std::shared_ptr getInstance() + { + return global_storage_pool; + } + + GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); ~GlobalStoragePool(); + void restore(); + PageStoragePtr log() const { return log_storage; } PageStoragePtr data() const { return data_storage; } PageStoragePtr meta() const { return meta_storage; } @@ -58,6 +89,7 @@ class GlobalStoragePool : private boost::noncopyable bool gc(const Settings & settings, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); private: + static std::shared_ptr global_storage_pool; PageStoragePtr log_storage; PageStoragePtr data_storage; PageStoragePtr meta_storage; @@ -69,6 +101,7 @@ class GlobalStoragePool : private boost::noncopyable Context & global_context; BackgroundProcessingPool::TaskHandle gc_handle; }; +using GlobalStoragePoolPtr = std::shared_ptr; class StoragePool : private boost::noncopyable { @@ -77,9 +110,7 @@ class StoragePool : private boost::noncopyable using Timepoint = Clock::time_point; using Seconds = std::chrono::seconds; - StoragePool(const String & name, NamespaceId ns_id_, StoragePathPool & path_pool, Context & global_ctx, const Settings & settings); - - StoragePool(NamespaceId ns_id_, const GlobalStoragePool & global_storage_pool, Context & global_ctx); + StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & path_pool, Context & global_ctx, const String & name = ""); void restore(); @@ -87,25 +118,27 @@ class StoragePool : private boost::noncopyable NamespaceId getNamespaceId() const { return ns_id; } - PageStoragePtr log() const { return log_storage; } - PageStoragePtr data() const { return data_storage; } - PageStoragePtr meta() const { return meta_storage; } + // TODO remove + PageStoragePtr log() const { return log_storage_v2; } + PageStoragePtr data() const { return data_storage_v2; } + PageStoragePtr meta() const { return meta_storage_v2; } - PageReader & logReader() { return log_storage_reader; } - PageReader & dataReader() { return data_storage_reader; } - PageReader & metaReader() { return meta_storage_reader; } + PageReaderPtr & logReader() { return log_storage_reader; } + PageReaderPtr & dataReader() { return data_storage_reader; } + PageReaderPtr & metaReader() { return meta_storage_reader; } + // TODO change to two snapshot from both of v2 and v3 PageReader newLogReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, log_storage, snapshot_read ? log_storage->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(ns_id, log_storage_v2, snapshot_read ? log_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); } PageReader newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, data_storage, snapshot_read ? data_storage->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(ns_id, data_storage_v2, snapshot_read ? data_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); } PageReader newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, meta_storage, snapshot_read ? meta_storage->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(ns_id, meta_storage_v2, snapshot_read ? meta_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); } void enableGC(); @@ -125,17 +158,22 @@ class StoragePool : private boost::noncopyable PageId newMetaPageId() { return ++max_meta_page_id; } private: + StoragePoolRunMode run_mode = StoragePoolRunMode::ONLY_V2; + // whether the three storage instance is owned by this StoragePool - const bool owned_storage = false; const NamespaceId ns_id; - const PageStoragePtr log_storage; - const PageStoragePtr data_storage; - const PageStoragePtr meta_storage; + PageStoragePtr log_storage_v2; + PageStoragePtr data_storage_v2; + PageStoragePtr meta_storage_v2; + + PageStoragePtr log_storage_v3; + PageStoragePtr data_storage_v3; + PageStoragePtr meta_storage_v3; - PageReader log_storage_reader; - PageReader data_storage_reader; - PageReader meta_storage_reader; + PageReaderPtr log_storage_reader; + PageReaderPtr data_storage_reader; + PageReaderPtr meta_storage_reader; std::atomic last_try_gc_time = Clock::now(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp index 93b366f3b15..f46fcf6bf18 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp @@ -84,7 +84,7 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique("test.t1", table_id, *storage_path_pool, *db_context, db_context->getSettingsRef()); + storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, table_id, nullptr, *storage_path_pool, *db_context, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index e6fb6f236f3..070578ffacf 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -105,7 +105,7 @@ class DMFile_Test parent_path = TiFlashStorageTestBasic::getTemporaryPath(); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFile_Test", false)); - storage_pool = std::make_unique("test.t1", /*table_id*/ 100, *path_pool, *db_context, db_context->getSettingsRef()); + storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); dm_file = DMFile::create(1, parent_path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); @@ -1057,7 +1057,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic auto configuration = mode == DMFileMode::DirectoryChecksum ? std::make_optional() : std::nullopt; path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique("test.t1", table_id, *path_pool, *db_context, DB::Settings()); + storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, table_id, nullptr, *path_pool, *db_context, "test.t1"); dm_file = DMFile::create(0, path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 197ec73fdb5..2fca10d6059 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -74,7 +74,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique("test.t1", /*table_id*/ 100, *storage_path_pool, *db_context, db_context->getSettingsRef()); + storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *storage_path_pool, *db_context, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index 8fc5c145f81..ad56099d8e8 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -53,7 +53,7 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique("test.t1", /*table_id*/ 100, *path_pool, *db_context, db_context->getSettingsRef()); + storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*table_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); storage_pool->restore(); if (!cols) cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 501a749aaa1..4f53c9cb2af 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -323,6 +323,7 @@ class PageReader : private boost::noncopyable , storage(storage_) , read_limiter(read_limiter_) {} + /// Snapshot read. PageReader(NamespaceId ns_id_, PageStoragePtr storage_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) : ns_id(ns_id_) @@ -384,6 +385,27 @@ class PageReader : private boost::noncopyable PageStorage::SnapshotPtr snap; ReadLimiterPtr read_limiter; }; +using PageReaderPtr = std::shared_ptr; + +class PageWriter : private boost::noncopyable +{ +public: + PageWriter(PageStoragePtr storage_, WriteLimiterPtr write_limiter_) + : storage(storage_) + , write_limiter(write_limiter_) + { + } + + void write(WriteBatch && write_batch) + { + storage->write(std::move(write_batch), write_limiter); + } + +private: + PageStoragePtr storage; + WriteLimiterPtr write_limiter; +}; +using PageWriterPtr = std::shared_ptr; } // namespace DB diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index b5b8b62b7ca..b97c74dee06 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -166,20 +167,11 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, auto delegator = path_pool.getPSDiskDelegatorRaft(); auto provider = global_context.getFileProvider(); // If the GlobalStoragePool is initialized, then use v3 format - bool use_v3_format = global_context.getGlobalStoragePool() != nullptr; - if (use_v3_format) + const auto & run_mode = global_context.getStoragePoolRunMode(); + // todo + switch (run_mode) { - mergeConfigFromSettings(global_context.getSettingsRef(), config); - - LOG_FMT_INFO(log, "RegionPersister running in v3 mode"); - page_storage = std::make_unique( // - "RegionPersister", - delegator, - config, - provider); - page_storage->restore(); - } - else + case DM::StoragePoolRunMode::ONLY_V2: { // If there is no PageFile with basic version binary format, use version 2 of PageStorage. auto detect_binary_version = DB::PS::V2::PageStorage::getMaxDataVersion(provider, delegator); @@ -211,6 +203,26 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, c, provider); } + break; + } + case DM::StoragePoolRunMode::ONLY_V3: + { + mergeConfigFromSettings(global_context.getSettingsRef(), config); + + LOG_FMT_INFO(log, "RegionPersister running in v3 mode"); + page_storage = std::make_unique( // + "RegionPersister", + delegator, + config, + provider); + page_storage->restore(); + break; + } + case DM::StoragePoolRunMode::MIX_MODE: + { + // TODO + break; + } } } diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index 870db4217c4..d72e052a6cc 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -73,8 +74,8 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ raft_config.disable_bg_flush = true; global_context->createTMTContext(raft_config, pingcap::ClusterConfig()); - if (global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool(), enable_ps_v3)) - LOG_FMT_INFO(&Poco::Logger::get("TiFlashTestEnv"), "PageStorage V3 enabled."); + global_context->initializeStoragePoolMode(global_context->getPathPool(), enable_ps_v3); + LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getStoragePoolRunMode())); global_context->setDeltaIndexManager(1024 * 1024 * 100 /*100MB*/); From 4d7e3e133f2923db536c640dfa09c2f5fd57d62b Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 21 Apr 2022 14:13:19 +0800 Subject: [PATCH 02/38] changed interface Signed-off-by: jiaqizho --- dbms/src/Interpreters/Context.cpp | 10 +- dbms/src/Interpreters/Context.h | 6 +- dbms/src/Server/DTTool/DTToolBench.cpp | 2 +- dbms/src/Server/tests/gtest_dttool.cpp | 2 +- dbms/src/Server/tests/gtest_server_config.cpp | 6 +- .../DeltaMerge/Delta/DeltaValueSpace.cpp | 4 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 20 +- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 119 +++++++----- dbms/src/Storages/DeltaMerge/StoragePool.h | 98 +++++++--- dbms/src/Storages/DeltaMerge/WriteBatches.h | 16 +- .../tests/gtest_dm_delta_value_space.cpp | 2 +- .../DeltaMerge/tests/gtest_dm_file.cpp | 4 +- .../DeltaMerge/tests/gtest_dm_segment.cpp | 2 +- .../tests/gtest_dm_segment_common_handle.cpp | 2 +- dbms/src/Storages/Page/PageStorage.h | 178 +++++++++++++++--- dbms/src/Storages/Page/Snapshot.h | 16 ++ .../Storages/Transaction/RegionPersister.cpp | 8 +- dbms/src/TestUtils/TiFlashTestEnv.cpp | 4 +- 18 files changed, 350 insertions(+), 149 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index da2717f78b4..aa096fec3d8 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -161,7 +161,7 @@ struct ContextShared PathCapacityMetricsPtr path_capacity_ptr; /// Path capacity metrics FileProviderPtr file_provider; /// File provider. IORateLimiter io_rate_limiter; - DM::StoragePoolRunMode storage_run_mode; + PageStorageRunMode storage_run_mode; /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. class SessionKeyHash @@ -1567,19 +1567,19 @@ static bool isPageStorageV2Existed(const PathPool & path_pool) return true; } -void Context::initializeStoragePoolMode(const PathPool & path_pool, bool enable_ps_v3) +void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3) { auto lock = getLock(); if (!enable_ps_v3) { - shared->storage_run_mode = DM::StoragePoolRunMode::ONLY_V2; + shared->storage_run_mode = PageStorageRunMode::ONLY_V2; } - shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? DM::StoragePoolRunMode::MIX_MODE : DM::StoragePoolRunMode::ONLY_V3; + shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; } -DM::StoragePoolRunMode Context::getStoragePoolRunMode() const +PageStorageRunMode Context::getPageStorageRunMode() const { auto lock = getLock(); return shared->storage_run_mode; diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index d81f1ee9370..5e255e95069 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -99,11 +99,11 @@ using WriteLimiterPtr = std::shared_ptr; class ReadLimiter; using ReadLimiterPtr = std::shared_ptr; +enum class PageStorageRunMode : UInt8; namespace DM { class MinMaxIndexCache; class DeltaIndexManager; -enum class StoragePoolRunMode : UInt8; } // namespace DM /// (database name, table name) @@ -404,8 +404,8 @@ class Context ReadLimiterPtr getReadLimiter() const; IORateLimiter & getIORateLimiter() const; - void initializeStoragePoolMode(const PathPool & path_pool, bool enable_ps_v3); - DM::StoragePoolRunMode getStoragePoolRunMode() const; + void initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3); + PageStorageRunMode getPageStorageRunMode() const; /// Call after initialization before using system logs. Call for global context. void initializeSystemLogs(); diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index 82053e93ef6..3639a29d712 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -336,7 +336,7 @@ int benchEntry(const std::vector & opts) auto settings = DB::Settings(); auto db_context = env.getContext(); auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); + auto storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index b79edb69c33..6d1a550d5b1 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -72,7 +72,7 @@ struct DTToolTest : public DB::base::TiFlashStorageTestBasic properties.push_back(property); } auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); + auto storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index 46aae7e4740..415551edaba 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -363,12 +363,12 @@ dt_page_gc_low_write_prob = 0.2 auto & global_ctx = TiFlashTestEnv::getGlobalContext(); std::unique_ptr path_pool = std::make_unique(global_ctx.getPathPool().withTable("test", "t1", false)); - std::unique_ptr storage_pool = std::make_unique(DM::StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, global_ctx, "test.t1"); + std::unique_ptr storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, global_ctx, "test.t1"); auto verify_storage_pool_reload_config = [&global_ctx](std::unique_ptr & storage_pool) { DB::Settings & settings = global_ctx.getSettingsRef(); - auto cfg = storage_pool->data()->getSettings(); + auto cfg = storage_pool->data_storage_v2->getSettings(); EXPECT_NE(cfg.gc_min_files, settings.dt_storage_pool_data_gc_min_file_num); EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num); EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes); @@ -379,7 +379,7 @@ dt_page_gc_low_write_prob = 0.2 storage_pool->gc(settings, DM::StoragePool::Seconds(0)); - cfg = storage_pool->data()->getSettings(); + cfg = storage_pool->data_storage_v2->getSettings(); EXPECT_EQ(cfg.gc_min_files, settings.dt_storage_pool_data_gc_min_file_num); EXPECT_EQ(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num); EXPECT_EQ(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes); diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp index b57ce3cb50a..132732d6989 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp @@ -236,12 +236,12 @@ bool DeltaValueSpace::compact(DMContext & context) LOG_FMT_DEBUG(log, "{} Nothing to compact", simpleInfo()); return true; } - log_storage_snap = context.storage_pool.log()->getSnapshot(/*tracing_id*/ fmt::format("minor_compact_{}", simpleInfo())); + log_storage_snap = context.storage_pool.logReader()->getSnapshot(/*tracing_id*/ fmt::format("minor_compact_{}", simpleInfo())); } // do compaction task WriteBatches wbs(context.storage_pool, context.getWriteLimiter()); - PageReader reader(context.storage_pool.getNamespaceId(), context.storage_pool.log(), std::move(log_storage_snap), context.getReadLimiter()); + const auto & reader = context.storage_pool.newLogReader(context.getReadLimiter(), log_storage_snap); compaction_task->prepare(context, wbs, reader); { diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 272c09512bc..3241cc0b1da 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -215,8 +215,8 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; - const auto & storage_pool_run_mode = global_context.getStoragePoolRunMode(); - if (storage_pool_run_mode == StoragePoolRunMode::ONLY_V3 || storage_pool_run_mode == StoragePoolRunMode::MIX_MODE) + const auto & storage_pool_run_mode = global_context.getPageStorageRunMode(); + if (storage_pool_run_mode == PageStorageRunMode::ONLY_V3 || storage_pool_run_mode == PageStorageRunMode::MIX_MODE) { GlobalStoragePool::init(global_context.getPathPool(), global_context, db_context.getSettingsRef()); } @@ -335,7 +335,7 @@ void DeltaMergeStore::setUpBackgroundTask(const DMContextPtr & dm_context) }; callbacks.ns_id = storage_pool->getNamespaceId(); // remember to unregister it when shutdown - storage_pool->data()->registerExternalPagesCallbacks(callbacks); + storage_pool->dataRegisterExternalPagesCallbacks(callbacks); storage_pool->enableGC(); background_task_handle = background_pool.addTask([this] { return handleBackgroundTask(false); }); @@ -439,7 +439,7 @@ void DeltaMergeStore::shutdown() // shutdown before unregister to avoid conflict between this thread and background gc thread on the `ExternalPagesCallbacks` // because PageStorage V2 doesn't have any lock protection on the `ExternalPagesCallbacks`.(The order doesn't matter for V3) storage_pool->shutdown(); - storage_pool->data()->unregisterExternalPagesCallbacks(storage_pool->getNamespaceId()); + storage_pool->dataUnregisterExternalPagesCallbacks(storage_pool->getNamespaceId()); background_pool.removeTask(background_task_handle); blockable_background_pool.removeTask(blockable_background_pool_handle); @@ -2345,12 +2345,12 @@ DeltaMergeStoreStat DeltaMergeStore::getStat() static const String useless_tracing_id("DeltaMergeStore::getStat"); { - auto snaps_stat = storage_pool->data()->getSnapshotsStat(); + auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat(); stat.storage_stable_num_snapshots = snaps_stat.num_snapshots; stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr stable_snapshot = storage_pool->data()->getSnapshot(useless_tracing_id); + PageStorage::SnapshotPtr stable_snapshot = storage_pool->dataReader()->getSnapshot(useless_tracing_id); const auto * concrete_snap = toConcreteSnapshot(stable_snapshot); if (concrete_snap) { @@ -2367,12 +2367,12 @@ DeltaMergeStoreStat DeltaMergeStore::getStat() } } { - auto snaps_stat = storage_pool->log()->getSnapshotsStat(); + auto snaps_stat = storage_pool->logReader()->getSnapshotsStat(); stat.storage_delta_num_snapshots = snaps_stat.num_snapshots; stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr log_snapshot = storage_pool->log()->getSnapshot(useless_tracing_id); + PageStorage::SnapshotPtr log_snapshot = storage_pool->logReader()->getSnapshot(useless_tracing_id); const auto * concrete_snap = toConcreteSnapshot(log_snapshot); if (concrete_snap) { @@ -2389,12 +2389,12 @@ DeltaMergeStoreStat DeltaMergeStore::getStat() } } { - auto snaps_stat = storage_pool->meta()->getSnapshotsStat(); + auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat(); stat.storage_meta_num_snapshots = snaps_stat.num_snapshots; stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - PageStorage::SnapshotPtr meta_snapshot = storage_pool->meta()->getSnapshot(useless_tracing_id); + PageStorage::SnapshotPtr meta_snapshot = storage_pool->metaReader()->getSnapshot(useless_tracing_id); const auto * concrete_snap = toConcreteSnapshot(meta_snapshot); if (concrete_snap) { diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index b1af99fb9a4..1bd5e6f503f 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -65,26 +65,6 @@ PageStorage::Config extractConfig(const Settings & settings, StorageType subtype return config; } -template -static bool doStoragePoolGC(const Context & global_context, const Settings & settings, const T & storage_pool) -{ - bool done_anything = false; - auto write_limiter = global_context.getWriteLimiter(); - auto read_limiter = global_context.getReadLimiter(); - auto config = extractConfig(settings, StorageType::Meta); - storage_pool.meta()->reloadSettings(config); - done_anything |= storage_pool.meta()->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Data); - storage_pool.data()->reloadSettings(config); - done_anything |= storage_pool.data()->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Log); - storage_pool.log()->reloadSettings(config); - done_anything |= storage_pool.log()->gc(/*not_skip*/ false, write_limiter, read_limiter); - - return done_anything; -} std::shared_ptr GlobalStoragePool::global_storage_pool = nullptr; GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings) @@ -145,17 +125,32 @@ bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_per last_try_gc_time = now; } - return doStoragePoolGC(global_context, settings, *this); + bool done_anything = false; + auto write_limiter = global_context.getWriteLimiter(); + auto read_limiter = global_context.getReadLimiter(); + auto config = extractConfig(settings, StorageType::Meta); + meta_storage->reloadSettings(config); + done_anything |= meta_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Data); + data_storage->reloadSettings(config); + done_anything |= data_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Log); + log_storage->reloadSettings(config); + done_anything |= log_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + return done_anything; } -StoragePool::StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & storage_path_pool, Context & global_ctx, const String & name) +StoragePool::StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & storage_path_pool, Context & global_ctx, const String & name) : run_mode(mode) , ns_id(ns_id_) , global_context(global_ctx) { switch (run_mode) { - case StoragePoolRunMode::ONLY_V2: + case PageStorageRunMode::ONLY_V2: { log_storage_v2 = PageStorage::create(name + ".log", storage_path_pool.getPSDiskDelegatorMulti("log"), @@ -169,30 +164,38 @@ StoragePool::StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const Glob storage_path_pool.getPSDiskDelegatorMulti("meta"), extractConfig(global_context.getSettingsRef(), StorageType::Meta), global_ctx.getFileProvider()); - log_storage_reader = std::make_shared(ns_id, log_storage_v2, nullptr); - data_storage_reader = std::make_shared(ns_id, data_storage_v2, nullptr); - meta_storage_reader = std::make_shared(ns_id, meta_storage_v2, nullptr); + log_storage_reader = std::make_shared(run_mode, ns_id, log_storage_v2, /*storage_v3_*/ nullptr, nullptr); + data_storage_reader = std::make_shared(run_mode, ns_id, data_storage_v2, /*storage_v3_*/ nullptr, nullptr); + meta_storage_reader = std::make_shared(run_mode, ns_id, meta_storage_v2, /*storage_v3_*/ nullptr, nullptr); + + log_storage_writer = std::make_shared(run_mode, log_storage_v2, /*storage_v3_*/ nullptr); + data_storage_writer = std::make_shared(run_mode, data_storage_v2, /*storage_v3_*/ nullptr); + meta_storage_writer = std::make_shared(run_mode, meta_storage_v2, /*storage_v3_*/ nullptr); break; } - case StoragePoolRunMode::ONLY_V3: + case PageStorageRunMode::ONLY_V3: { assert(global_storage_pool != nullptr); - log_storage_v3 = global_storage_pool->log(); - data_storage_v3 = global_storage_pool->data(); - meta_storage_v3 = global_storage_pool->meta(); + log_storage_v3 = global_storage_pool->log_storage; + data_storage_v3 = global_storage_pool->data_storage; + meta_storage_v3 = global_storage_pool->meta_storage; - log_storage_reader = std::make_shared(ns_id, log_storage_v3, nullptr); - data_storage_reader = std::make_shared(ns_id, data_storage_v3, nullptr); - meta_storage_reader = std::make_shared(ns_id, meta_storage_v3, nullptr); + log_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, log_storage_v3, nullptr); + data_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, data_storage_v3, nullptr); + meta_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, meta_storage_v3, nullptr); + + log_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, log_storage_v3); + data_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, data_storage_v3); + meta_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, meta_storage_v3); break; } - case StoragePoolRunMode::MIX_MODE: + case PageStorageRunMode::MIX_MODE: { assert(global_storage_pool != nullptr); - log_storage_v3 = global_storage_pool->log(); - data_storage_v3 = global_storage_pool->data(); - meta_storage_v3 = global_storage_pool->meta(); + log_storage_v3 = global_storage_pool->log_storage; + data_storage_v3 = global_storage_pool->data_storage; + meta_storage_v3 = global_storage_pool->meta_storage; log_storage_v2 = PageStorage::create(name + ".log", storage_path_pool.getPSDiskDelegatorMulti("log"), @@ -207,21 +210,24 @@ StoragePool::StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const Glob extractConfig(global_context.getSettingsRef(), StorageType::Meta), global_ctx.getFileProvider()); - // log_storage_reader = std::make_shared(ns_id, log_storage, nullptr); - // data_storage_reader = std::make_shared(ns_id, data_storage, nullptr); - // meta_storage_reader = std::make_shared(ns_id, meta_storage, nullptr); + log_storage_reader = std::make_shared(run_mode, ns_id, log_storage_v2, log_storage_v3, nullptr); + data_storage_reader = std::make_shared(run_mode, ns_id, data_storage_v2, data_storage_v3, nullptr); + meta_storage_reader = std::make_shared(run_mode, ns_id, meta_storage_v2, meta_storage_v3, nullptr); + log_storage_writer = std::make_shared(run_mode, log_storage_v2, log_storage_v3); + data_storage_writer = std::make_shared(run_mode, data_storage_v2, data_storage_v3); + meta_storage_writer = std::make_shared(run_mode, meta_storage_v2, meta_storage_v3); break; } default: - throw Exception(fmt::format("Unknown StoragePoolRunMode {}", static_cast(run_mode))); + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode))); } } void StoragePool::restore() { // If the storage instances is not global, we need to initialize it by ourselves and add a gc task. - if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) + if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) { log_storage_v2->restore(); data_storage_v2->restore(); @@ -230,21 +236,21 @@ void StoragePool::restore() switch (run_mode) { - case StoragePoolRunMode::ONLY_V2: + case PageStorageRunMode::ONLY_V2: { max_log_page_id = log_storage_v2->getMaxId(ns_id); max_data_page_id = data_storage_v2->getMaxId(ns_id); max_meta_page_id = meta_storage_v2->getMaxId(ns_id); break; } - case StoragePoolRunMode::ONLY_V3: + case PageStorageRunMode::ONLY_V3: { max_log_page_id = log_storage_v3->getMaxId(ns_id); max_data_page_id = data_storage_v3->getMaxId(ns_id); max_meta_page_id = meta_storage_v3->getMaxId(ns_id); break; } - case StoragePoolRunMode::MIX_MODE: + case PageStorageRunMode::MIX_MODE: { max_log_page_id = std::max(log_storage_v2->getMaxId(ns_id), log_storage_v3->getMaxId(ns_id)); max_data_page_id = std::max(data_storage_v2->getMaxId(ns_id), data_storage_v3->getMaxId(ns_id)); @@ -261,7 +267,7 @@ StoragePool::~StoragePool() void StoragePool::enableGC() { - if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) + if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) { gc_handle = global_context.getBackgroundPool().addTask([this] { return this->gc(global_context.getSettingsRef()); }); } @@ -269,7 +275,7 @@ void StoragePool::enableGC() bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { - if (run_mode == StoragePoolRunMode::ONLY_V3) + if (run_mode == PageStorageRunMode::ONLY_V3) return false; { @@ -283,7 +289,22 @@ bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) last_try_gc_time = now; } - return doStoragePoolGC(global_context, settings, *this); + // Only do the v2 GC + bool done_anything = false; + auto write_limiter = global_context.getWriteLimiter(); + auto read_limiter = global_context.getReadLimiter(); + auto config = extractConfig(settings, StorageType::Meta); + meta_storage_v2->reloadSettings(config); + done_anything |= meta_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Data); + data_storage_v2->reloadSettings(config); + done_anything |= data_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Log); + log_storage_v2->reloadSettings(config); + done_anything |= log_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + return done_anything; } void StoragePool::shutdown() @@ -299,7 +320,7 @@ void StoragePool::drop() { shutdown(); - if (run_mode == StoragePoolRunMode::ONLY_V2 || run_mode == StoragePoolRunMode::MIX_MODE) + if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) { meta_storage_v2->drop(); data_storage_v2->drop(); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index e986a6b7b59..48d67702549 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -35,13 +35,6 @@ using StoragePoolPtr = std::shared_ptr; static const std::chrono::seconds DELTA_MERGE_GC_PERIOD(60); -enum class StoragePoolRunMode : UInt8 -{ - ONLY_V2 = 1, - ONLY_V3 = 2, - MIX_MODE = 3, -}; - class GlobalStoragePool : private boost::noncopyable { public: @@ -80,9 +73,7 @@ class GlobalStoragePool : private boost::noncopyable void restore(); - PageStoragePtr log() const { return log_storage; } - PageStoragePtr data() const { return data_storage; } - PageStoragePtr meta() const { return meta_storage; } + friend class StoragePool; private: // TODO: maybe more frequent gc for GlobalStoragePool? @@ -110,7 +101,7 @@ class StoragePool : private boost::noncopyable using Timepoint = Clock::time_point; using Seconds = std::chrono::seconds; - StoragePool(StoragePoolRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & path_pool, Context & global_ctx, const String & name = ""); + StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & path_pool, Context & global_ctx, const String & name = ""); void restore(); @@ -118,31 +109,84 @@ class StoragePool : private boost::noncopyable NamespaceId getNamespaceId() const { return ns_id; } - // TODO remove - PageStoragePtr log() const { return log_storage_v2; } - PageStoragePtr data() const { return data_storage_v2; } - PageStoragePtr meta() const { return meta_storage_v2; } + PageReaderPtr & logReader() + { + assert(log_storage_reader != nullptr); + return log_storage_reader; + } + + PageReaderPtr & dataReader() + { + assert(data_storage_reader != nullptr); + return data_storage_reader; + } + + PageReaderPtr & metaReader() + { + assert(meta_storage_reader != nullptr); + return meta_storage_reader; + } + + PageWriterPtr & logWriter() + { + assert(log_storage_writer != nullptr); + return log_storage_writer; + } + + PageWriterPtr & dataWriter() + { + assert(data_storage_writer != nullptr); + return data_storage_writer; + } - PageReaderPtr & logReader() { return log_storage_reader; } - PageReaderPtr & dataReader() { return data_storage_reader; } - PageReaderPtr & metaReader() { return meta_storage_reader; } + PageWriterPtr & metaWriter() + { + assert(meta_storage_writer != nullptr); + return meta_storage_writer; + } - // TODO change to two snapshot from both of v2 and v3 PageReader newLogReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, log_storage_v2, snapshot_read ? log_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(run_mode, ns_id, log_storage_v2, log_storage_v3, snapshot_read ? log_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); } + + PageReader newLogReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) + { + return PageReader(run_mode, ns_id, log_storage_v2, log_storage_v3, snapshot, read_limiter); + } + PageReader newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, data_storage_v2, snapshot_read ? data_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(run_mode, ns_id, data_storage_v2, data_storage_v3, snapshot_read ? data_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + } + + PageReader newDataReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) + { + return PageReader(run_mode, ns_id, data_storage_v2, data_storage_v3, snapshot, read_limiter); } + PageReader newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { - return PageReader(ns_id, meta_storage_v2, snapshot_read ? meta_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + return PageReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, snapshot_read ? meta_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + } + + PageReader newMetaReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) + { + return PageReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, snapshot, read_limiter); } void enableGC(); + void dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks & callbacks) + { + data_storage_v2->registerExternalPagesCallbacks(callbacks); + } + + void dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) + { + data_storage_v2->unregisterExternalPagesCallbacks(ns_id); + } + bool gc(const Settings & settings, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); void shutdown(); @@ -153,12 +197,12 @@ class StoragePool : private boost::noncopyable PageId newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who); PageId maxMetaPageId() { return max_meta_page_id; } - PageId newLogPageId() { return ++max_log_page_id; } PageId newMetaPageId() { return ++max_meta_page_id; } - +#ifndef DBMS_PUBLIC_GTEST private: - StoragePoolRunMode run_mode = StoragePoolRunMode::ONLY_V2; +#endif + const PageStorageRunMode run_mode; // whether the three storage instance is owned by this StoragePool const NamespaceId ns_id; @@ -175,6 +219,10 @@ class StoragePool : private boost::noncopyable PageReaderPtr data_storage_reader; PageReaderPtr meta_storage_reader; + PageWriterPtr log_storage_writer; + PageWriterPtr data_storage_writer; + PageWriterPtr meta_storage_writer; + std::atomic last_try_gc_time = Clock::now(); std::mutex mutex; diff --git a/dbms/src/Storages/DeltaMerge/WriteBatches.h b/dbms/src/Storages/DeltaMerge/WriteBatches.h index c64ff2cde38..2b508fed068 100644 --- a/dbms/src/Storages/DeltaMerge/WriteBatches.h +++ b/dbms/src/Storages/DeltaMerge/WriteBatches.h @@ -112,8 +112,8 @@ struct WriteBatches : private boost::noncopyable for (auto & w : data.getWrites()) data_write_pages.push_back(w.page_id); - storage_pool.log()->write(std::move(log), write_limiter); - storage_pool.data()->write(std::move(data), write_limiter); + storage_pool.logWriter()->write(std::move(log), write_limiter); + storage_pool.dataWriter()->write(std::move(data), write_limiter); for (auto page_id : log_write_pages) written_log.push_back(page_id); @@ -152,8 +152,8 @@ struct WriteBatches : private boost::noncopyable check(data_wb, "data_wb", logger); } - storage_pool.log()->write(std::move(log_wb), write_limiter); - storage_pool.data()->write(std::move(data_wb), write_limiter); + storage_pool.logWriter()->write(std::move(log_wb), write_limiter); + storage_pool.dataWriter()->write(std::move(data_wb), write_limiter); written_log.clear(); written_data.clear(); @@ -179,7 +179,7 @@ struct WriteBatches : private boost::noncopyable check(meta, "meta", logger); } - storage_pool.meta()->write(std::move(meta), write_limiter); + storage_pool.metaWriter()->write(std::move(meta), write_limiter); meta.clear(); } @@ -205,9 +205,9 @@ struct WriteBatches : private boost::noncopyable check(removed_meta, "removed_meta", logger); } - storage_pool.log()->write(std::move(removed_log), write_limiter); - storage_pool.data()->write(std::move(removed_data), write_limiter); - storage_pool.meta()->write(std::move(removed_meta), write_limiter); + storage_pool.logWriter()->write(std::move(removed_log), write_limiter); + storage_pool.dataWriter()->write(std::move(removed_data), write_limiter); + storage_pool.metaWriter()->write(std::move(removed_meta), write_limiter); removed_log.clear(); removed_data.clear(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp index f46fcf6bf18..57abe397e63 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp @@ -84,7 +84,7 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, table_id, nullptr, *storage_path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, table_id, nullptr, *storage_path_pool, *db_context, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index 070578ffacf..a58e977926e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -105,7 +105,7 @@ class DMFile_Test parent_path = TiFlashStorageTestBasic::getTemporaryPath(); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFile_Test", false)); - storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); dm_file = DMFile::create(1, parent_path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); @@ -1057,7 +1057,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic auto configuration = mode == DMFileMode::DirectoryChecksum ? std::make_optional() : std::nullopt; path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, table_id, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, table_id, nullptr, *path_pool, *db_context, "test.t1"); dm_file = DMFile::create(0, path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 2fca10d6059..e7f73386bb8 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -74,7 +74,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *storage_path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *storage_path_pool, *db_context, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index ad56099d8e8..e2d6ef0d20c 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -53,7 +53,7 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique(StoragePoolRunMode::ONLY_V2, /*table_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*table_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); storage_pool->restore(); if (!cols) cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 4f53c9cb2af..92ad1b16a04 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -46,6 +46,14 @@ class Context; class PageStorage; using PageStoragePtr = std::shared_ptr; + +enum class PageStorageRunMode : UInt8 +{ + ONLY_V2 = 1, + ONLY_V3 = 2, + MIX_MODE = 3, +}; + struct ExternalPageCallbacks { // `scanner` for scanning avaliable external page ids on disks. @@ -318,70 +326,149 @@ class PageReader : private boost::noncopyable { public: /// Not snapshot read. - explicit PageReader(NamespaceId ns_id_, PageStoragePtr storage_, ReadLimiterPtr read_limiter_) - : ns_id(ns_id_) - , storage(storage_) + explicit PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, ReadLimiterPtr read_limiter_) + : run_mode(run_mode_) + , ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) , read_limiter(read_limiter_) - {} + { + (void)run_mode; + (void)storage_v3; + } /// Snapshot read. - PageReader(NamespaceId ns_id_, PageStoragePtr storage_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) - : ns_id(ns_id_) - , storage(storage_) + PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) + : run_mode(run_mode_) + , ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) , snap(snap_) , read_limiter(read_limiter_) - {} - PageReader(NamespaceId ns_id_, PageStoragePtr storage_, PageStorage::SnapshotPtr && snap_, ReadLimiterPtr read_limiter_) - : ns_id(ns_id_) - , storage(storage_) + { + (void)run_mode; + (void)storage_v3; + } + + PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr && snap_, ReadLimiterPtr read_limiter_) + : run_mode(run_mode_) + , ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) , snap(std::move(snap_)) , read_limiter(read_limiter_) - {} + { + (void)run_mode; + (void)storage_v3; + } DB::Page read(PageId page_id) const { - return storage->read(ns_id, page_id, read_limiter, snap); + return selectPageStorage()->read(ns_id, page_id, read_limiter, snap); } PageMap read(const std::vector & page_ids) const { - return storage->read(ns_id, page_ids, read_limiter, snap); + return selectPageStorage()->read(ns_id, page_ids, read_limiter, snap); } void read(const std::vector & page_ids, PageHandler & handler) const { - storage->read(ns_id, page_ids, handler, read_limiter, snap); + selectPageStorage()->read(ns_id, page_ids, handler, read_limiter, snap); } using PageReadFields = PageStorage::PageReadFields; PageMap read(const std::vector & page_fields) const { - return storage->read(ns_id, page_fields, read_limiter, snap); + return selectPageStorage()->read(ns_id, page_fields, read_limiter, snap); } - PageId getMaxId() const - { - return storage->getMaxId(ns_id); - } + // PageId getMaxId() const + // { + // return storage_v2->getMaxId(ns_id); + // } PageId getNormalPageId(PageId page_id) const { - return storage->getNormalPageId(ns_id, page_id, snap); + return storage_v2->getNormalPageId(ns_id, page_id, snap); } UInt64 getPageChecksum(PageId page_id) const { - return storage->getEntry(ns_id, page_id, snap).checksum; + return storage_v2->getEntry(ns_id, page_id, snap).checksum; } PageEntry getPageEntry(PageId page_id) const { - return storage->getEntry(ns_id, page_id, snap); + return storage_v2->getEntry(ns_id, page_id, snap); + } + + PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getSnapshot(tracing_id); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getSnapshot(tracing_id); + } + case PageStorageRunMode::MIX_MODE: + { + PageStorageSnapshotMixed a(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // + storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); + (void)a; + + return storage_v2->getSnapshot(tracing_id); + // return std::shared_ptr(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // + // storage_v3->getSnapshot(fmt::format("{}-v3",tracing_id))); + } + } + return storage_v2->getSnapshot(tracing_id); + } + + // Get some statistics of all living snapshots and the oldest living snapshot. + SnapshotsStatistics getSnapshotsStat() const + { + return storage_v2->getSnapshotsStat(); } private: + PageStoragePtr selectPageStorage() const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + assert(storage_v2); + return storage_v2; + } + case PageStorageRunMode::ONLY_V3: + { + assert(storage_v3); + return storage_v3; + } + case PageStorageRunMode::MIX_MODE: + { + assert(storage_v2); + assert(storage_v3); + + // todo + return storage_v3; + } + default: + //todo + return storage_v3; + } + } + +private: + const PageStorageRunMode run_mode; NamespaceId ns_id; - PageStoragePtr storage; + PageStoragePtr storage_v2; + PageStoragePtr storage_v3; PageStorage::SnapshotPtr snap; ReadLimiterPtr read_limiter; }; @@ -390,20 +477,49 @@ using PageReaderPtr = std::shared_ptr; class PageWriter : private boost::noncopyable { public: - PageWriter(PageStoragePtr storage_, WriteLimiterPtr write_limiter_) - : storage(storage_) - , write_limiter(write_limiter_) + PageWriter(PageStorageRunMode run_mode_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_) + : run_mode(run_mode_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) { + (void)run_mode; + (void)storage_v3; } - void write(WriteBatch && write_batch) + void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) { - storage->write(std::move(write_batch), write_limiter); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + assert(storage_v2); + storage_v2->write(std::move(write_batch), write_limiter); + break; + } + case PageStorageRunMode::ONLY_V3: + { + assert(storage_v3); + storage_v3->write(std::move(write_batch), write_limiter); + break; + } + case PageStorageRunMode::MIX_MODE: + { + assert(storage_v2); + assert(storage_v3); + + for (const auto & write : write_batch.getWrites()) + { + (void)write; + } + break; + } + } } private: - PageStoragePtr storage; - WriteLimiterPtr write_limiter; + PageStorageRunMode run_mode; + PageStoragePtr storage_v2; + PageStoragePtr storage_v3; }; using PageWriterPtr = std::shared_ptr; diff --git a/dbms/src/Storages/Page/Snapshot.h b/dbms/src/Storages/Page/Snapshot.h index 348d084382f..f8003c04d40 100644 --- a/dbms/src/Storages/Page/Snapshot.h +++ b/dbms/src/Storages/Page/Snapshot.h @@ -33,4 +33,20 @@ class PageStorageSnapshot }; using PageStorageSnapshotPtr = std::shared_ptr; +class PageStorageSnapshotMixed : public PageStorageSnapshot +{ +public: + // TODO: add/sub CurrentMetrics::PSMVCCNumSnapshots in here + PageStorageSnapshotMixed(PageStorageSnapshotPtr snapshot_v2_, PageStorageSnapshotPtr snapshot_v3_) + : snapshot_v2(snapshot_v2_) + , snapshot_v3(snapshot_v3_) + {} + + ~PageStorageSnapshotMixed(){}; + +private: + PageStorageSnapshotPtr snapshot_v2; + PageStorageSnapshotPtr snapshot_v3; +}; + } // namespace DB diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index b97c74dee06..b8a01c74773 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -167,11 +167,11 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, auto delegator = path_pool.getPSDiskDelegatorRaft(); auto provider = global_context.getFileProvider(); // If the GlobalStoragePool is initialized, then use v3 format - const auto & run_mode = global_context.getStoragePoolRunMode(); + const auto & run_mode = global_context.getPageStorageRunMode(); // todo switch (run_mode) { - case DM::StoragePoolRunMode::ONLY_V2: + case PageStorageRunMode::ONLY_V2: { // If there is no PageFile with basic version binary format, use version 2 of PageStorage. auto detect_binary_version = DB::PS::V2::PageStorage::getMaxDataVersion(provider, delegator); @@ -205,7 +205,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, } break; } - case DM::StoragePoolRunMode::ONLY_V3: + case PageStorageRunMode::ONLY_V3: { mergeConfigFromSettings(global_context.getSettingsRef(), config); @@ -218,7 +218,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, page_storage->restore(); break; } - case DM::StoragePoolRunMode::MIX_MODE: + case PageStorageRunMode::MIX_MODE: { // TODO break; diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index d72e052a6cc..e1cb6623c73 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -74,8 +74,8 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ raft_config.disable_bg_flush = true; global_context->createTMTContext(raft_config, pingcap::ClusterConfig()); - global_context->initializeStoragePoolMode(global_context->getPathPool(), enable_ps_v3); - LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getStoragePoolRunMode())); + global_context->initializePageStorageMode(global_context->getPathPool(), enable_ps_v3); + LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); global_context->setDeltaIndexManager(1024 * 1024 * 100 /*100MB*/); From a2dc3c995a08c81de2897e150791d54b34094f06 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 22 Apr 2022 14:15:48 +0800 Subject: [PATCH 03/38] add interface --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 9 +- dbms/src/Storages/Page/Page.h | 14 + dbms/src/Storages/Page/PageStorage.h | 295 +++++++++++++++---- dbms/src/Storages/Page/Snapshot.h | 20 +- dbms/src/Storages/Page/WriteBatch.h | 5 + 5 files changed, 289 insertions(+), 54 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 1bd5e6f503f..5d8f4b564b6 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -21,6 +21,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} // namespace ErrorCodes + namespace FailPoints { extern const char force_set_dtfile_exist_when_acquire_id[]; @@ -58,7 +63,7 @@ PageStorage::Config extractConfig(const Settings & settings, StorageType subtype SET_CONFIG(meta); break; default: - throw Exception("Unknown subtype in extractConfig: " + DB::toString(static_cast(subtype))); + throw Exception(fmt::format("Unknown subtype in extractConfig: {} ", static_cast(subtype)), ErrorCodes::LOGICAL_ERROR); } #undef SET_CONFIG @@ -220,7 +225,7 @@ StoragePool::StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const Glob break; } default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode))); + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } } diff --git a/dbms/src/Storages/Page/Page.h b/dbms/src/Storages/Page/Page.h index b36bf7245d6..c461f4b7b0b 100644 --- a/dbms/src/Storages/Page/Page.h +++ b/dbms/src/Storages/Page/Page.h @@ -76,6 +76,20 @@ struct Page return ByteBuffer(data.begin() + beg, data.begin() + end); } + inline static PageFieldSizes fieldOffsetsToSizes(std::set & field_offsets, size_t data_size) + { + PageFieldSizes field_size = {}; + + auto it = field_offsets.begin(); + while (it != field_offsets.end()) + { + PageFieldOffset beg = it->offset; + ++it; + field_size.emplace_back(it == field_offsets.end() ? data_size - beg : it->offset - beg); + } + return field_size; + } + size_t fieldSize() const { return field_offsets.size(); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 92ad1b16a04..6daba980408 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -46,6 +46,11 @@ class Context; class PageStorage; using PageStoragePtr = std::shared_ptr; +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} // namespace ErrorCodes + enum class PageStorageRunMode : UInt8 { @@ -321,7 +326,6 @@ class PageStorage : private boost::noncopyable FileProviderPtr file_provider; }; - class PageReader : private boost::noncopyable { public: @@ -333,8 +337,6 @@ class PageReader : private boost::noncopyable , storage_v3(storage_v3_) , read_limiter(read_limiter_) { - (void)run_mode; - (void)storage_v3; } /// Snapshot read. @@ -346,8 +348,6 @@ class PageReader : private boost::noncopyable , snap(snap_) , read_limiter(read_limiter_) { - (void)run_mode; - (void)storage_v3; } PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr && snap_, ReadLimiterPtr read_limiter_) @@ -358,49 +358,173 @@ class PageReader : private boost::noncopyable , snap(std::move(snap_)) , read_limiter(read_limiter_) { - (void)run_mode; - (void)storage_v3; } DB::Page read(PageId page_id) const { - return selectPageStorage()->read(ns_id, page_id, read_limiter, snap); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->read(ns_id, page_id, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->read(ns_id, page_id, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + if (page_from_v3.page_id == INVALID_PAGE_ID) + { + return storage_v2->read(ns_id, page_id, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } PageMap read(const std::vector & page_ids) const { - return selectPageStorage()->read(ns_id, page_ids, read_limiter, snap); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->read(ns_id, page_ids, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->read(ns_id, page_ids, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + auto it = page_maps.begin(); + while (it != page_maps.end()) + { + if (it->second.page_id == INVALID_PAGE_ID) + { + it->second = storage_v2->read(ns_id, it->first, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + it++; + } + return page_maps; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } void read(const std::vector & page_ids, PageHandler & handler) const { - selectPageStorage()->read(ns_id, page_ids, handler, read_limiter, snap); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->read(ns_id, page_ids, handler, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->read(ns_id, page_ids, handler, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } using PageReadFields = PageStorage::PageReadFields; PageMap read(const std::vector & page_fields) const { - return selectPageStorage()->read(ns_id, page_fields, read_limiter, snap); - } + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->read(ns_id, page_fields, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->read(ns_id, page_fields, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + + for (auto page_field : page_fields) + { + if (page_maps[page_field.first].page_id == INVALID_PAGE_ID) + { + page_maps[page_field.first] = storage_v2->read(ns_id, page_field, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + } - // PageId getMaxId() const - // { - // return storage_v2->getMaxId(ns_id); - // } + return page_maps; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } + } PageId getNormalPageId(PageId page_id) const { - return storage_v2->getNormalPageId(ns_id, page_id, snap); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getNormalPageId(ns_id, page_id, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getNormalPageId(ns_id, page_id, snap); + } + case PageStorageRunMode::MIX_MODE: + { + PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + if (resolved_page_id != INVALID_PAGE_ID) + { + return resolved_page_id; + } + return storage_v2->getNormalPageId(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } UInt64 getPageChecksum(PageId page_id) const { - return storage_v2->getEntry(ns_id, page_id, snap).checksum; + return getPageEntry(page_id).checksum; } PageEntry getPageEntry(PageId page_id) const { - return storage_v2->getEntry(ns_id, page_id, snap); + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getEntry(ns_id, page_id, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getEntry(ns_id, page_id, snap); + } + case PageStorageRunMode::MIX_MODE: + { + PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV3Snasphot()); + if (page_entry.file_id != INVALID_BLOBFILE_ID) + { + return page_entry; + } + return storage_v2->getEntry(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const @@ -417,50 +541,51 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - PageStorageSnapshotMixed a(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // - storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); - (void)a; - - return storage_v2->getSnapshot(tracing_id); - // return std::shared_ptr(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // - // storage_v3->getSnapshot(fmt::format("{}-v3",tracing_id))); + return std::make_shared(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // + storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } - return storage_v2->getSnapshot(tracing_id); } // Get some statistics of all living snapshots and the oldest living snapshot. SnapshotsStatistics getSnapshotsStat() const - { - return storage_v2->getSnapshotsStat(); - } - -private: - PageStoragePtr selectPageStorage() const { switch (run_mode) { case PageStorageRunMode::ONLY_V2: { - assert(storage_v2); - return storage_v2; + return storage_v2->getSnapshotsStat(); } case PageStorageRunMode::ONLY_V3: { - assert(storage_v3); - return storage_v3; + return storage_v3->getSnapshotsStat(); } case PageStorageRunMode::MIX_MODE: { - assert(storage_v2); - assert(storage_v3); + SnapshotsStatistics statistics_total; + const auto & statistics_from_v2 = storage_v2->getSnapshotsStat(); + const auto & statistics_from_v3 = storage_v3->getSnapshotsStat(); - // todo - return storage_v3; + statistics_total.num_snapshots = statistics_from_v2.num_snapshots + statistics_from_v3.num_snapshots; + if (statistics_from_v2.longest_living_seconds > statistics_from_v3.longest_living_seconds) + { + statistics_total.longest_living_seconds = statistics_from_v2.longest_living_seconds; + statistics_total.longest_living_from_thread_id = statistics_from_v2.longest_living_from_thread_id; + statistics_total.longest_living_from_tracing_id = statistics_from_v2.longest_living_from_tracing_id; + } + else + { + statistics_total.longest_living_seconds = statistics_from_v3.longest_living_seconds; + statistics_total.longest_living_from_thread_id = statistics_from_v3.longest_living_from_thread_id; + statistics_total.longest_living_from_tracing_id = statistics_from_v3.longest_living_from_tracing_id; + } + + return statistics_total; } default: - //todo - return storage_v3; + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } } @@ -482,8 +607,6 @@ class PageWriter : private boost::noncopyable , storage_v2(storage_v2_) , storage_v3(storage_v3_) { - (void)run_mode; - (void)storage_v3; } void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) @@ -492,24 +615,94 @@ class PageWriter : private boost::noncopyable { case PageStorageRunMode::ONLY_V2: { - assert(storage_v2); storage_v2->write(std::move(write_batch), write_limiter); break; } case PageStorageRunMode::ONLY_V3: { - assert(storage_v3); storage_v3->write(std::move(write_batch), write_limiter); break; } case PageStorageRunMode::MIX_MODE: { - assert(storage_v2); - assert(storage_v3); - + const auto & ns_id = write_batch.getNamespaceId(); + WriteBatch wb_for_v2{write_batch.getNamespaceId()}; for (const auto & write : write_batch.getWrites()) { - (void)write; + switch (write.type) + { + // PUT/PUT_EXTERNAL only for V3 + case WriteBatch::WriteType::PUT: + case WriteBatch::WriteType::PUT_EXTERNAL: + { + break; + } + // Both need del in v2 and v3 + // TODO: make sure we can del not existed pageid + case WriteBatch::WriteType::DEL: + { + wb_for_v2.copyWrite(write); + break; + } + case WriteBatch::WriteType::REF: + { + PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, + write.ori_page_id, + /*snapshot*/ nullptr, + false); + + // if normal id is not ok, read from v2 and create a new put + ref + if (resolved_page_id == INVALID_PAGE_ID) + { + const auto & entry_for_put = storage_v2->getEntry(ns_id, write.ori_page_id, /*snapshot*/ {}); + if (entry_for_put.file_id != 0) + { + WriteBatch wb_for_put{ns_id}; + auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); + + // Page with fields + if (entry_for_put.field_offsets.size() != 0) + { + wb_for_put.putPage(write.ori_page_id, // + 0, + std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), + page_for_put.data.size(), + Page::fieldOffsetsToSizes(page_for_put.field_offsets, + page_for_put.data.size())); + } + else + { // Normal page with fields + wb_for_put.putPage(write.ori_page_id, // + 0, + std::make_shared(page_for_put.data.begin(), + page_for_put.data.size()), + page_for_put.data.size()); + } + storage_v2->write(std::move(wb_for_put), write_limiter); + } + else + { + throw Exception(fmt::format("Can't find origin entry in V2 and V3, [ns_id={}, ori_page_id={}]", + ns_id, + write.ori_page_id), + ErrorCodes::LOGICAL_ERROR); + } + } + // else V3 found the origin one. + // Then do nothing. + break; + } + default: + { + throw Exception(fmt::format("Unknown write type: {}", write.type)); + } + } + } + + storage_v3->write(std::move(write_batch), write_limiter); + if (!wb_for_v2.empty()) + { + storage_v2->write(std::move(wb_for_v2), write_limiter); } break; } diff --git a/dbms/src/Storages/Page/Snapshot.h b/dbms/src/Storages/Page/Snapshot.h index f8003c04d40..7d3da27b9f3 100644 --- a/dbms/src/Storages/Page/Snapshot.h +++ b/dbms/src/Storages/Page/Snapshot.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include @@ -37,16 +38,33 @@ class PageStorageSnapshotMixed : public PageStorageSnapshot { public: // TODO: add/sub CurrentMetrics::PSMVCCNumSnapshots in here - PageStorageSnapshotMixed(PageStorageSnapshotPtr snapshot_v2_, PageStorageSnapshotPtr snapshot_v3_) + PageStorageSnapshotMixed(const PageStorageSnapshotPtr & snapshot_v2_, const PageStorageSnapshotPtr & snapshot_v3_) : snapshot_v2(snapshot_v2_) , snapshot_v3(snapshot_v3_) {} ~PageStorageSnapshotMixed(){}; + PageStorageSnapshotPtr getV2Snasphot() + { + return snapshot_v2; + } + + PageStorageSnapshotPtr getV3Snasphot() + { + return snapshot_v3; + } + private: PageStorageSnapshotPtr snapshot_v2; PageStorageSnapshotPtr snapshot_v3; }; +using PageStorageSnapshotMixedPtr = std::shared_ptr; + +static inline PageStorageSnapshotMixedPtr +toConcreteMixedSnapshot(const PageStorageSnapshotPtr & ptr) +{ + return std::static_pointer_cast(ptr); +} } // namespace DB diff --git a/dbms/src/Storages/Page/WriteBatch.h b/dbms/src/Storages/Page/WriteBatch.h index bde03c4de57..419f88e08ed 100644 --- a/dbms/src/Storages/Page/WriteBatch.h +++ b/dbms/src/Storages/Page/WriteBatch.h @@ -184,6 +184,11 @@ class WriteBatch : private boost::noncopyable std::swap(o.sequence, sequence); } + void copyWrite(const Write write) + { + writes.emplace_back(write); + } + void clear() { Writes tmp; From 0e367ffd07c87e41702c9d30d984caf0e4d97703 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sun, 24 Apr 2022 16:56:22 +0800 Subject: [PATCH 04/38] change RegionPersister.cpp --- dbms/src/Server/Server.cpp | 11 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 8 +- dbms/src/Storages/Page/PageStorage.h | 112 +++++++++++++++--- dbms/src/Storages/Page/Snapshot.h | 2 +- .../Storages/Transaction/RegionPersister.cpp | 51 +++++--- .../Storages/Transaction/RegionPersister.h | 4 +- 6 files changed, 142 insertions(+), 46 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 8187b7ef487..f5689797c37 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1112,15 +1112,8 @@ int Server::main(const std::vector & /*args*/) raft_config.enable_compatible_mode, // global_context->getPathCapacity(), global_context->getFileProvider()); - // must initialize before the following operation: - // 1. load data from disk(because this process may depend on the initialization of global StoragePool) - // 2. initialize KVStore service - // 1) because we need to check whether this is the first startup of this node, and we judge it based on whether there are any files in kvstore directory - // 2) KVStore service also choose its data format based on whether the GlobalStoragePool is initialized - - // TODO - // if (global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool(), storage_config.enable_ps_v3)) - // LOG_FMT_INFO(log, "PageStorage V3 enabled."); + + global_context->initializePageStorageMode(global_context->getPathPool(), storage_config.enable_ps_v3); // Use pd address to define which default_database we use by default. // For mock test, we use "default". For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 3241cc0b1da..5969eecc5f8 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -210,18 +210,18 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, , hash_salt(++DELTA_MERGE_STORE_HASH_SALT) , log(Logger::get("DeltaMergeStore", fmt::format("{}.{}", db_name, table_name))) { - LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}]", db_name, table_name); + const auto & page_storage_run_mode = global_context.getPageStorageRunMode(); + LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}] [ps_run_mode={}]", db_name, table_name, static_cast(page_storage_run_mode)); // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; - const auto & storage_pool_run_mode = global_context.getPageStorageRunMode(); - if (storage_pool_run_mode == PageStorageRunMode::ONLY_V3 || storage_pool_run_mode == PageStorageRunMode::MIX_MODE) + if (page_storage_run_mode == PageStorageRunMode::ONLY_V3 || page_storage_run_mode == PageStorageRunMode::MIX_MODE) { GlobalStoragePool::init(global_context.getPathPool(), global_context, db_context.getSettingsRef()); } - storage_pool = std::make_shared(storage_pool_run_mode, + storage_pool = std::make_shared(page_storage_run_mode, ns_id, GlobalStoragePool::getInstance(), path_pool, diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 6daba980408..1715bc072c1 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -45,6 +45,7 @@ using PSDiskDelegatorPtr = std::shared_ptr; class Context; class PageStorage; using PageStoragePtr = std::shared_ptr; +class RegionPersister; namespace ErrorCodes { @@ -374,10 +375,10 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); if (page_from_v3.page_id == INVALID_PAGE_ID) { - return storage_v2->read(ns_id, page_id, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); } } default: @@ -399,13 +400,13 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteV3Snapshot(), false); auto it = page_maps.begin(); while (it != page_maps.end()) { if (it->second.page_id == INVALID_PAGE_ID) { - it->second = storage_v2->read(ns_id, it->first, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + it->second = storage_v2->read(ns_id, it->first, read_limiter, toConcreteV2Snapshot()); } it++; } @@ -430,8 +431,8 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); - storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteV3Snapshot(), false); + storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteV2Snapshot()); } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -453,13 +454,13 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteV3Snapshot(), false); - for (auto page_field : page_fields) + for (const auto & page_field : page_fields) { if (page_maps[page_field.first].page_id == INVALID_PAGE_ID) { - page_maps[page_field.first] = storage_v2->read(ns_id, page_field, read_limiter, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + page_maps[page_field.first] = storage_v2->read(ns_id, page_field, read_limiter, toConcreteV2Snapshot()); } } @@ -484,12 +485,12 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV3Snasphot(), false); + PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteV3Snapshot(), false); if (resolved_page_id != INVALID_PAGE_ID) { return resolved_page_id; } - return storage_v2->getNormalPageId(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + return storage_v2->getNormalPageId(ns_id, page_id, toConcreteV2Snapshot()); } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -515,12 +516,12 @@ class PageReader : private boost::noncopyable } case PageStorageRunMode::MIX_MODE: { - PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV3Snasphot()); + PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteV3Snapshot()); if (page_entry.file_id != INVALID_BLOBFILE_ID) { return page_entry; } - return storage_v2->getEntry(ns_id, page_id, toConcreteMixedSnapshot(snap)->getV2Snasphot()); + return storage_v2->getEntry(ns_id, page_id, toConcreteV2Snapshot()); } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -589,6 +590,39 @@ class PageReader : private boost::noncopyable } } + void traverse(const std::function & acceptor) const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->traverse(acceptor, snap); + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->traverse(acceptor, snap); + } + case PageStorageRunMode::MIX_MODE: + { + storage_v2->traverse(acceptor, toConcreteV3Snapshot()); + storage_v3->traverse(acceptor, toConcreteV2Snapshot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } + } + +private: + PageStorage::SnapshotPtr toConcreteV3Snapshot() const + { + return snap ? toConcreteMixedSnapshot(snap)->getV3Snasphot() : snap; + } + + PageStorage::SnapshotPtr toConcreteV2Snapshot() const + { + return snap ? toConcreteMixedSnapshot(snap)->getV2Snasphot() : snap; + } + private: const PageStorageRunMode run_mode; NamespaceId ns_id; @@ -609,7 +643,7 @@ class PageWriter : private boost::noncopyable { } - void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) + void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const { switch (run_mode) { @@ -661,7 +695,7 @@ class PageWriter : private boost::noncopyable auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); // Page with fields - if (entry_for_put.field_offsets.size() != 0) + if (!entry_for_put.field_offsets.empty()) { wb_for_put.putPage(write.ori_page_id, // 0, @@ -709,6 +743,54 @@ class PageWriter : private boost::noncopyable } } + friend class RegionPersister; + +private: + void reloadSettings(const PageStorage::Config & new_config) const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->reloadSettings(new_config); + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->reloadSettings(new_config); + } + case PageStorageRunMode::MIX_MODE: + { + storage_v2->reloadSettings(new_config); + storage_v3->reloadSettings(new_config); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } + }; + + bool gc(bool not_skip, const WriteLimiterPtr & write_limiter, const ReadLimiterPtr & read_limiter) const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->gc(not_skip, write_limiter, read_limiter); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->gc(not_skip, write_limiter, read_limiter); + } + case PageStorageRunMode::MIX_MODE: + { + bool ok = storage_v2->gc(not_skip, write_limiter, read_limiter); + ok |= storage_v3->gc(not_skip, write_limiter, read_limiter); + return ok; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } + } + private: PageStorageRunMode run_mode; PageStoragePtr storage_v2; diff --git a/dbms/src/Storages/Page/Snapshot.h b/dbms/src/Storages/Page/Snapshot.h index 7d3da27b9f3..622707e8fdf 100644 --- a/dbms/src/Storages/Page/Snapshot.h +++ b/dbms/src/Storages/Page/Snapshot.h @@ -43,7 +43,7 @@ class PageStorageSnapshotMixed : public PageStorageSnapshot , snapshot_v3(snapshot_v3_) {} - ~PageStorageSnapshotMixed(){}; + ~PageStorageSnapshotMixed() = default; PageStorageSnapshotPtr getV2Snasphot() { diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index b8a01c74773..cedab357c70 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -43,11 +43,11 @@ extern const char force_disable_region_persister_compatible_mode[]; void RegionPersister::drop(RegionID region_id, const RegionTaskLock &) { - if (page_storage) + if (page_writer) { DB::WriteBatch wb_v2{ns_id}; wb_v2.delPage(region_id); - page_storage->write(std::move(wb_v2), global_context.getWriteLimiter()); + page_writer->write(std::move(wb_v2), global_context.getWriteLimiter()); } else { @@ -102,9 +102,9 @@ void RegionPersister::doPersist(RegionCacheWriteElement & region_write_buffer, c std::lock_guard lock(mutex); - if (page_storage) + if (page_reader) { - auto entry = page_storage->getEntry(ns_id, region_id, nullptr); + auto entry = page_reader->getPageEntry(region_id); if (entry.isValid() && entry.tag > applied_index) return; } @@ -122,11 +122,11 @@ void RegionPersister::doPersist(RegionCacheWriteElement & region_write_buffer, c } auto read_buf = buffer.tryGetReadBuffer(); - if (page_storage) + if (page_writer) { DB::WriteBatch wb{ns_id}; wb.putPage(region_id, applied_index, read_buf, region_size); - page_storage->write(std::move(wb), global_context.getWriteLimiter()); + page_writer->write(std::move(wb), global_context.getWriteLimiter()); } else { @@ -186,12 +186,14 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, config.num_write_slots = 4; // extend write slots to 4 at least LOG_FMT_INFO(log, "RegionPersister running in v2 mode"); - page_storage = std::make_unique( + auto page_storage_v2 = std::make_shared( "RegionPersister", delegator, config, provider); - page_storage->restore(); + page_storage_v2->restore(); + page_writer = std::make_shared(run_mode, page_storage_v2, /*storage_v3_*/ nullptr); + page_reader = std::make_shared(run_mode, ns_id, page_storage_v2, /*storage_v3_*/ nullptr, /*readlimiter*/ global_context.getReadLimiter()); } else { @@ -210,24 +212,41 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, mergeConfigFromSettings(global_context.getSettingsRef(), config); LOG_FMT_INFO(log, "RegionPersister running in v3 mode"); - page_storage = std::make_unique( // + auto page_storage_v3 = std::make_shared( // "RegionPersister", delegator, config, provider); - page_storage->restore(); + page_storage_v3->restore(); + page_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, page_storage_v3); + page_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, page_storage_v3, global_context.getReadLimiter()); break; } case PageStorageRunMode::MIX_MODE: { - // TODO + LOG_FMT_INFO(log, "RegionPersister running in mix mode"); + auto page_storage_v2 = std::make_shared( + "RegionPersister", + delegator, + config, + provider); + auto page_storage_v3 = std::make_shared( // + "RegionPersister", + delegator, + config, + provider); + + page_storage_v2->restore(); + page_storage_v3->restore(); + page_writer = std::make_shared(run_mode, page_storage_v2, page_storage_v3); + page_reader = std::make_shared(run_mode, ns_id, page_storage_v2, page_storage_v3, global_context.getReadLimiter()); break; } } } RegionMap regions; - if (page_storage) + if (page_reader) { auto acceptor = [&](const DB::Page & page) { ReadBufferFromMemory buf(page.data.begin(), page.data.size()); @@ -236,7 +255,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, throw Exception("region id and page id not match!", ErrorCodes::LOGICAL_ERROR); regions.emplace(page.page_id, region); }; - page_storage->traverse(acceptor, nullptr); + page_reader->traverse(acceptor); } else { @@ -255,11 +274,11 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, bool RegionPersister::gc() { - if (page_storage) + if (page_writer) { PageStorage::Config config = getConfigFromSettings(global_context.getSettingsRef()); - page_storage->reloadSettings(config); - return page_storage->gc(false, nullptr, nullptr); + page_writer->reloadSettings(config); + return page_writer->gc(false, nullptr, nullptr); } else return stable_page_storage->gc(); diff --git a/dbms/src/Storages/Transaction/RegionPersister.h b/dbms/src/Storages/Transaction/RegionPersister.h index feb4353a0d0..8c72bdb87c2 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.h +++ b/dbms/src/Storages/Transaction/RegionPersister.h @@ -67,7 +67,9 @@ class RegionPersister final : private boost::noncopyable #endif Context & global_context; - PageStoragePtr page_storage; + PageWriterPtr page_writer; + PageReaderPtr page_reader; + std::shared_ptr stable_page_storage; // RegionPersister stores it's data individually, so the `ns_id` value doesn't matter From f84a85edbc5595902714482f81b1888954f12747 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sun, 24 Apr 2022 17:04:23 +0800 Subject: [PATCH 05/38] add number of valid page check --- dbms/src/Interpreters/Context.cpp | 6 +++ dbms/src/Interpreters/Context.h | 1 + dbms/src/Server/tests/gtest_server_config.cpp | 5 +-- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 9 ++--- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 22 +++++++++- dbms/src/Storages/DeltaMerge/StoragePool.h | 10 ++++- dbms/src/Storages/Page/PageStorage.h | 40 ++++++++++++++++--- .../Storages/Transaction/RegionPersister.cpp | 38 +++++++++++++----- .../Storages/Transaction/RegionPersister.h | 6 ++- .../tests/gtest_region_persister.cpp | 12 ++++-- dbms/src/TestUtils/TiFlashTestEnv.cpp | 6 +-- dbms/src/TestUtils/TiFlashTestEnv.h | 2 +- 12 files changed, 120 insertions(+), 37 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index aa096fec3d8..627a84c9398 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1585,6 +1585,12 @@ PageStorageRunMode Context::getPageStorageRunMode() const return shared->storage_run_mode; } +void Context::setPageStorageRunMode(PageStorageRunMode run_mode) const +{ + auto lock = getLock(); + shared->storage_run_mode = run_mode; +} + UInt16 Context::getTCPPort() const { auto lock = getLock(); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 5e255e95069..93435a69ba5 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -405,6 +405,7 @@ class Context IORateLimiter & getIORateLimiter() const; void initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3); + void setPageStorageRunMode(PageStorageRunMode run_mode) const; PageStorageRunMode getPageStorageRunMode() const; /// Call after initialization before using system logs. Call for global context. diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index 415551edaba..e3ba77949af 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -293,7 +293,7 @@ dt_page_gc_low_write_prob = 0.2 auto verify_persister_reload_config = [&global_ctx](RegionPersister & persister) { DB::Settings & settings = global_ctx.getSettingsRef(); - auto cfg = persister.page_storage->getSettings(); + auto cfg = persister.getPageStorageSettings(); EXPECT_NE(cfg.gc_min_files, settings.dt_storage_pool_data_gc_min_file_num); EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num); EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes); @@ -303,8 +303,7 @@ dt_page_gc_low_write_prob = 0.2 EXPECT_NE(cfg.prob_do_gc_when_write_is_low, settings.dt_page_gc_low_write_prob * 1000); persister.gc(); - cfg = persister.page_storage->getSettings(); - + cfg = persister.getPageStorageSettings(); EXPECT_NE(cfg.gc_min_files, settings.dt_storage_pool_data_gc_min_file_num); EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num); EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 5969eecc5f8..d7bfc6427b7 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -210,8 +210,8 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, , hash_salt(++DELTA_MERGE_STORE_HASH_SALT) , log(Logger::get("DeltaMergeStore", fmt::format("{}.{}", db_name, table_name))) { - const auto & page_storage_run_mode = global_context.getPageStorageRunMode(); - LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}] [ps_run_mode={}]", db_name, table_name, static_cast(page_storage_run_mode)); + auto page_storage_run_mode = global_context.getPageStorageRunMode(); + LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}]", db_name, table_name); // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; @@ -245,10 +245,9 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, store_columns = generateStoreColumns(original_table_columns, is_common_handle); auto dm_context = newDMContext(db_context, db_context.getSettingsRef()); - try { - storage_pool->restore(); // restore from disk + page_storage_run_mode = storage_pool->restore(); // restore from disk if (!storage_pool->maxMetaPageId()) { // Create the first segment. @@ -281,7 +280,7 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, setUpBackgroundTask(dm_context); - LOG_FMT_INFO(log, "Restore DeltaMerge Store end [{}.{}]", db_name, table_name); + LOG_FMT_INFO(log, "Restore DeltaMerge Store end [{}.{}], [ps_run_mode={}]", db_name, table_name, static_cast(page_storage_run_mode)); } DeltaMergeStore::~DeltaMergeStore() diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 5d8f4b564b6..34b3fb540b1 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -210,7 +210,7 @@ StoragePool::StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const Glob storage_path_pool.getPSDiskDelegatorSingle("data"), extractConfig(global_context.getSettingsRef(), StorageType::Data), global_ctx.getFileProvider()); - data_storage_v2 = PageStorage::create(name + ".meta", + meta_storage_v2 = PageStorage::create(name + ".meta", storage_path_pool.getPSDiskDelegatorMulti("meta"), extractConfig(global_context.getSettingsRef(), StorageType::Meta), global_ctx.getFileProvider()); @@ -229,7 +229,7 @@ StoragePool::StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const Glob } } -void StoragePool::restore() +PageStorageRunMode StoragePool::restore() { // If the storage instances is not global, we need to initialize it by ourselves and add a gc task. if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) @@ -260,9 +260,27 @@ void StoragePool::restore() max_log_page_id = std::max(log_storage_v2->getMaxId(ns_id), log_storage_v3->getMaxId(ns_id)); max_data_page_id = std::max(data_storage_v2->getMaxId(ns_id), data_storage_v3->getMaxId(ns_id)); max_meta_page_id = std::max(meta_storage_v2->getMaxId(ns_id), meta_storage_v3->getMaxId(ns_id)); + + // Check number of valid pages in v2 + if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) + { + run_mode = PageStorageRunMode::ONLY_V3; + log_storage_v2 = nullptr; + data_storage_v2 = nullptr; + meta_storage_v2 = nullptr; + + log_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, log_storage_v3, nullptr); + data_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, data_storage_v3, nullptr); + meta_storage_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, meta_storage_v3, nullptr); + + log_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, log_storage_v3); + data_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, data_storage_v3); + meta_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, meta_storage_v3); + } break; } } + return run_mode; } StoragePool::~StoragePool() diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 48d67702549..bb8d4808c5b 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -103,12 +103,17 @@ class StoragePool : private boost::noncopyable StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & path_pool, Context & global_ctx, const String & name = ""); - void restore(); + PageStorageRunMode restore(); ~StoragePool(); NamespaceId getNamespaceId() const { return ns_id; } + PageStorageRunMode getPageStorageRunMode() + { + return run_mode; + } + PageReaderPtr & logReader() { assert(log_storage_reader != nullptr); @@ -199,10 +204,11 @@ class StoragePool : private boost::noncopyable PageId maxMetaPageId() { return max_meta_page_id; } PageId newLogPageId() { return ++max_log_page_id; } PageId newMetaPageId() { return ++max_meta_page_id; } + #ifndef DBMS_PUBLIC_GTEST private: #endif - const PageStorageRunMode run_mode; + PageStorageRunMode run_mode; // whether the three storage instance is owned by this StoragePool const NamespaceId ns_id; diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 1715bc072c1..19c683363be 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -367,19 +367,20 @@ class PageReader : private boost::noncopyable { case PageStorageRunMode::ONLY_V2: { - storage_v2->read(ns_id, page_id, read_limiter, snap); + return storage_v2->read(ns_id, page_id, read_limiter, snap); } case PageStorageRunMode::ONLY_V3: { - storage_v3->read(ns_id, page_id, read_limiter, snap); + return storage_v3->read(ns_id, page_id, read_limiter, snap); } case PageStorageRunMode::MIX_MODE: { const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); - if (page_from_v3.page_id == INVALID_PAGE_ID) + if (page_from_v3.page_id != INVALID_PAGE_ID) { - return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); + return page_from_v3; } + return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -424,15 +425,18 @@ class PageReader : private boost::noncopyable case PageStorageRunMode::ONLY_V2: { storage_v2->read(ns_id, page_ids, handler, read_limiter, snap); + break; } case PageStorageRunMode::ONLY_V3: { storage_v3->read(ns_id, page_ids, handler, read_limiter, snap); + break; } case PageStorageRunMode::MIX_MODE: { const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteV3Snapshot(), false); storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteV2Snapshot()); + break; } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -597,15 +601,18 @@ class PageReader : private boost::noncopyable case PageStorageRunMode::ONLY_V2: { storage_v2->traverse(acceptor, snap); + break; } case PageStorageRunMode::ONLY_V3: { storage_v3->traverse(acceptor, snap); + break; } case PageStorageRunMode::MIX_MODE: { storage_v2->traverse(acceptor, toConcreteV3Snapshot()); storage_v3->traverse(acceptor, toConcreteV2Snapshot()); + break; } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); @@ -672,7 +679,6 @@ class PageWriter : private boost::noncopyable break; } // Both need del in v2 and v3 - // TODO: make sure we can del not existed pageid case WriteBatch::WriteType::DEL: { wb_for_v2.copyWrite(write); @@ -746,6 +752,27 @@ class PageWriter : private boost::noncopyable friend class RegionPersister; private: + PageStorage::Config getSettings() const + { + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getSettings(); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getSettings(); + } + case PageStorageRunMode::MIX_MODE: + { + throw Exception("Not support.", ErrorCodes::NOT_IMPLEMENTED); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } + } + void reloadSettings(const PageStorage::Config & new_config) const { switch (run_mode) @@ -753,15 +780,18 @@ class PageWriter : private boost::noncopyable case PageStorageRunMode::ONLY_V2: { storage_v2->reloadSettings(new_config); + break; } case PageStorageRunMode::ONLY_V3: { storage_v3->reloadSettings(new_config); + break; } case PageStorageRunMode::MIX_MODE: { storage_v2->reloadSettings(new_config); storage_v3->reloadSettings(new_config); + break; } default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index cedab357c70..66e46699d34 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -66,7 +66,7 @@ void RegionPersister::computeRegionWriteBuffer(const Region & region, RegionCach if (unlikely(region_size > static_cast(std::numeric_limits::max()))) { LOG_FMT_WARNING( - &Poco::Logger::get("RegionPersister"), + Logger::get("RegionPersister"), "Persisting big region: {} with data info: {}, serialized size {}", region.toString(), region.dataInfo(), @@ -139,9 +139,19 @@ void RegionPersister::doPersist(RegionCacheWriteElement & region_write_buffer, c RegionPersister::RegionPersister(Context & global_context_, const RegionManager & region_manager_) : global_context(global_context_) , region_manager(region_manager_) - , log(&Poco::Logger::get("RegionPersister")) + , log(Logger::get("RegionPersister")) {} +PageStorage::Config RegionPersister::getPageStorageSettings() const +{ + if (!page_writer) + { + throw Exception("Not support for PS v1", ErrorCodes::LOGICAL_ERROR); + } + + return page_writer->getSettings(); +} + PS::V1::PageStorage::Config getV1PSConfig(const PS::V2::PageStorage::Config & config) { PS::V1::PageStorage::Config c; @@ -167,8 +177,8 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, auto delegator = path_pool.getPSDiskDelegatorRaft(); auto provider = global_context.getFileProvider(); // If the GlobalStoragePool is initialized, then use v3 format - const auto & run_mode = global_context.getPageStorageRunMode(); - // todo + auto run_mode = global_context.getPageStorageRunMode(); + switch (run_mode) { case PageStorageRunMode::ONLY_V2: @@ -185,7 +195,6 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, mergeConfigFromSettings(global_context.getSettingsRef(), config); config.num_write_slots = 4; // extend write slots to 4 at least - LOG_FMT_INFO(log, "RegionPersister running in v2 mode"); auto page_storage_v2 = std::make_shared( "RegionPersister", delegator, @@ -211,7 +220,6 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, { mergeConfigFromSettings(global_context.getSettingsRef(), config); - LOG_FMT_INFO(log, "RegionPersister running in v3 mode"); auto page_storage_v3 = std::make_shared( // "RegionPersister", delegator, @@ -224,7 +232,6 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, } case PageStorageRunMode::MIX_MODE: { - LOG_FMT_INFO(log, "RegionPersister running in mix mode"); auto page_storage_v2 = std::make_shared( "RegionPersister", delegator, @@ -238,11 +245,24 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, page_storage_v2->restore(); page_storage_v3->restore(); - page_writer = std::make_shared(run_mode, page_storage_v2, page_storage_v3); - page_reader = std::make_shared(run_mode, ns_id, page_storage_v2, page_storage_v3, global_context.getReadLimiter()); + + if (page_storage_v2->getNumberOfPages() == 0) + { + page_storage_v2 = nullptr; + run_mode = PageStorageRunMode::ONLY_V3; + page_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, page_storage_v3); + page_reader = std::make_shared(run_mode, ns_id, /*storage_v2_*/ nullptr, page_storage_v3, global_context.getReadLimiter()); + } + else + { + page_writer = std::make_shared(run_mode, page_storage_v2, page_storage_v3); + page_reader = std::make_shared(run_mode, ns_id, page_storage_v2, page_storage_v3, global_context.getReadLimiter()); + } break; } } + + LOG_FMT_INFO(log, "RegionPersister running. Current Run Mode is {}", static_cast(run_mode)); } RegionMap regions; diff --git a/dbms/src/Storages/Transaction/RegionPersister.h b/dbms/src/Storages/Transaction/RegionPersister.h index 8c72bdb87c2..fe6d7d98cd8 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.h +++ b/dbms/src/Storages/Transaction/RegionPersister.h @@ -14,11 +14,11 @@ #pragma once +#include #include #include #include #include -#include namespace DB { @@ -55,6 +55,8 @@ class RegionPersister final : private boost::noncopyable using RegionCacheWriteElement = std::tuple; static void computeRegionWriteBuffer(const Region & region, RegionCacheWriteElement & region_write_buffer); + PageStorage::Config getPageStorageSettings() const; + #ifndef DBMS_PUBLIC_GTEST private: #endif @@ -76,6 +78,6 @@ class RegionPersister final : private boost::noncopyable NamespaceId ns_id = MAX_NAMESPACE_ID; const RegionManager & region_manager; std::mutex mutex; - Poco::Logger * log; + LoggerPtr log; }; } // namespace DB diff --git a/dbms/src/Storages/Transaction/tests/gtest_region_persister.cpp b/dbms/src/Storages/Transaction/tests/gtest_region_persister.cpp index 55813edcbeb..16a35f42da1 100644 --- a/dbms/src/Storages/Transaction/tests/gtest_region_persister.cpp +++ b/dbms/src/Storages/Transaction/tests/gtest_region_persister.cpp @@ -309,7 +309,8 @@ try // Force to run in compatible mode FailPointHelper::enableFailPoint(FailPoints::force_enable_region_persister_compatible_mode); persister.restore(nullptr, config); - ASSERT_EQ(persister.page_storage, nullptr); + ASSERT_EQ(persister.page_writer, nullptr); + ASSERT_EQ(persister.page_reader, nullptr); ASSERT_NE(persister.stable_page_storage, nullptr); for (size_t i = 0; i < region_num; ++i) @@ -330,7 +331,8 @@ try RegionPersister persister(ctx, region_manager); // restore normally, should run in compatible mode. RegionMap new_regions = persister.restore(nullptr, config); - ASSERT_EQ(persister.page_storage, nullptr); + ASSERT_EQ(persister.page_writer, nullptr); + ASSERT_EQ(persister.page_reader, nullptr); ASSERT_NE(persister.stable_page_storage, nullptr); // Try to read for (size_t i = 0; i < region_num; ++i) @@ -349,7 +351,8 @@ try // Force to run in normal mode FailPointHelper::enableFailPoint(FailPoints::force_disable_region_persister_compatible_mode); RegionMap new_regions = persister.restore(nullptr, config); - ASSERT_NE(persister.page_storage, nullptr); + ASSERT_NE(persister.page_writer, nullptr); + ASSERT_NE(persister.page_reader, nullptr); ASSERT_EQ(persister.stable_page_storage, nullptr); // Try to read for (size_t i = 0; i < region_num; ++i) @@ -379,7 +382,8 @@ try RegionPersister persister(ctx, region_manager); // Restore normally, should run in normal mode. RegionMap new_regions = persister.restore(nullptr, config); - ASSERT_NE(persister.page_storage, nullptr); + ASSERT_NE(persister.page_writer, nullptr); + ASSERT_NE(persister.page_reader, nullptr); ASSERT_EQ(persister.stable_page_storage, nullptr); // Try to read for (size_t i = 0; i < region_num + region_num_under_nromal_mode; ++i) diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index e1cb6623c73..f20afaa7461 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -28,7 +28,7 @@ namespace DB::tests { std::unique_ptr TiFlashTestEnv::global_context = nullptr; -void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ps_v3) +void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path) { // set itself as global context global_context = std::make_unique(DB::Context::createGlobal()); @@ -74,9 +74,7 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ raft_config.disable_bg_flush = true; global_context->createTMTContext(raft_config, pingcap::ClusterConfig()); - global_context->initializePageStorageMode(global_context->getPathPool(), enable_ps_v3); - LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); - + global_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2); global_context->setDeltaIndexManager(1024 * 1024 * 100 /*100MB*/); global_context->getTMTContext().restore(); diff --git a/dbms/src/TestUtils/TiFlashTestEnv.h b/dbms/src/TestUtils/TiFlashTestEnv.h index 65dad63d937..a53b9ff7f52 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.h +++ b/dbms/src/TestUtils/TiFlashTestEnv.h @@ -88,7 +88,7 @@ class TiFlashTestEnv static Context getContext(const DB::Settings & settings = DB::Settings(), Strings testdata_path = {}); - static void initializeGlobalContext(Strings testdata_path = {}, bool enable_ps_v3 = false); + static void initializeGlobalContext(Strings testdata_path = {}); static Context & getGlobalContext() { return *global_context; } static void shutdown(); From fa993de40e16f725de049b59896521da41c590a9 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Mon, 25 Apr 2022 10:58:08 +0800 Subject: [PATCH 06/38] fix tests --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 61 ++++++++++++++++++++ dbms/src/Storages/DeltaMerge/StoragePool.h | 54 +++++------------ dbms/src/TestUtils/TiFlashTestEnv.cpp | 7 ++- dbms/src/TestUtils/TiFlashTestEnv.h | 2 +- 4 files changed, 81 insertions(+), 43 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 34b3fb540b1..865e0d52941 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -296,6 +296,18 @@ void StoragePool::enableGC() } } +void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks & callbacks) +{ + if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) + data_storage_v2->registerExternalPagesCallbacks(callbacks); +} + +void StoragePool::dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) +{ + if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) + data_storage_v2->unregisterExternalPagesCallbacks(ns_id); +} + bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { if (run_mode == PageStorageRunMode::ONLY_V3) @@ -383,5 +395,54 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons return dtfile_id; } +template +inline static PageReader newReader(const PageStorageRunMode run_mode, const NamespaceId ns_id, T & storage_v2, T & storage_v3, ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + return PageReader(run_mode, ns_id, storage_v2, nullptr, snapshot_read ? storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); + case PageStorageRunMode::ONLY_V3: + return PageReader(run_mode, ns_id, nullptr, storage_v3, snapshot_read ? storage_v3->getSnapshot(tracing_id) : nullptr, read_limiter); + case PageStorageRunMode::MIX_MODE: + return PageReader(run_mode, ns_id, storage_v2, storage_v3, snapshot_read ? std::make_shared(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // + storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))) + : nullptr, + read_limiter); + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +PageReader StoragePool::newLogReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) +{ + return newReader(run_mode, ns_id, log_storage_v2, log_storage_v3, read_limiter, snapshot_read, tracing_id); +} + +PageReader StoragePool::newLogReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) +{ + return PageReader(run_mode, ns_id, log_storage_v2, log_storage_v3, snapshot, read_limiter); +} + +PageReader StoragePool::newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) +{ + return newReader(run_mode, ns_id, data_storage_v2, data_storage_v3, read_limiter, snapshot_read, tracing_id); +} + +PageReader StoragePool::newDataReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) +{ + return PageReader(run_mode, ns_id, data_storage_v2, data_storage_v3, snapshot, read_limiter); +} + +PageReader StoragePool::newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) +{ + return newReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, read_limiter, snapshot_read, tracing_id); +} + +PageReader StoragePool::newMetaReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) +{ + return PageReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, snapshot, read_limiter); +} + } // namespace DM } // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index bb8d4808c5b..a4569cafd0c 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -116,81 +116,55 @@ class StoragePool : private boost::noncopyable PageReaderPtr & logReader() { - assert(log_storage_reader != nullptr); + assert(log_storage_reader); return log_storage_reader; } PageReaderPtr & dataReader() { - assert(data_storage_reader != nullptr); + assert(data_storage_reader); return data_storage_reader; } PageReaderPtr & metaReader() { - assert(meta_storage_reader != nullptr); + assert(meta_storage_reader); return meta_storage_reader; } PageWriterPtr & logWriter() { - assert(log_storage_writer != nullptr); + assert(log_storage_writer); return log_storage_writer; } PageWriterPtr & dataWriter() { - assert(data_storage_writer != nullptr); + assert(data_storage_writer); return data_storage_writer; } PageWriterPtr & metaWriter() { - assert(meta_storage_writer != nullptr); + assert(meta_storage_writer); return meta_storage_writer; } - PageReader newLogReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) - { - return PageReader(run_mode, ns_id, log_storage_v2, log_storage_v3, snapshot_read ? log_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); - } - - PageReader newLogReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) - { - return PageReader(run_mode, ns_id, log_storage_v2, log_storage_v3, snapshot, read_limiter); - } - PageReader newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) - { - return PageReader(run_mode, ns_id, data_storage_v2, data_storage_v3, snapshot_read ? data_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); - } - - PageReader newDataReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) - { - return PageReader(run_mode, ns_id, data_storage_v2, data_storage_v3, snapshot, read_limiter); - } + PageReader newLogReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id); + PageReader newLogReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot); - PageReader newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) - { - return PageReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, snapshot_read ? meta_storage_v2->getSnapshot(tracing_id) : nullptr, read_limiter); - } + PageReader newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id); + PageReader newDataReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot); - PageReader newMetaReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) - { - return PageReader(run_mode, ns_id, meta_storage_v2, meta_storage_v3, snapshot, read_limiter); - } + PageReader newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id); + PageReader newMetaReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot); void enableGC(); - void dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks & callbacks) - { - data_storage_v2->registerExternalPagesCallbacks(callbacks); - } + void dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks & callbacks); - void dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) - { - data_storage_v2->unregisterExternalPagesCallbacks(ns_id); - } + void dataUnregisterExternalPagesCallbacks(NamespaceId ns_id); bool gc(const Settings & settings, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index f20afaa7461..f7b1ba9b56c 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -28,7 +28,7 @@ namespace DB::tests { std::unique_ptr TiFlashTestEnv::global_context = nullptr; -void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path) +void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ps_v3) { // set itself as global context global_context = std::make_unique(DB::Context::createGlobal()); @@ -74,7 +74,10 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path) raft_config.disable_bg_flush = true; global_context->createTMTContext(raft_config, pingcap::ClusterConfig()); - global_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2); + + global_context->setPageStorageRunMode(enable_ps_v3 ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::ONLY_V2); + LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); + global_context->setDeltaIndexManager(1024 * 1024 * 100 /*100MB*/); global_context->getTMTContext().restore(); diff --git a/dbms/src/TestUtils/TiFlashTestEnv.h b/dbms/src/TestUtils/TiFlashTestEnv.h index a53b9ff7f52..65dad63d937 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.h +++ b/dbms/src/TestUtils/TiFlashTestEnv.h @@ -88,7 +88,7 @@ class TiFlashTestEnv static Context getContext(const DB::Settings & settings = DB::Settings(), Strings testdata_path = {}); - static void initializeGlobalContext(Strings testdata_path = {}); + static void initializeGlobalContext(Strings testdata_path = {}, bool enable_ps_v3 = false); static Context & getGlobalContext() { return *global_context; } static void shutdown(); From f13ed85b4742e9599c24e08df333e2341c75cd89 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Mon, 25 Apr 2022 12:12:47 +0800 Subject: [PATCH 07/38] fix dt_workload --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 10 +--------- dbms/src/Storages/DeltaMerge/StoragePool.h | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 865e0d52941..da145e86ce0 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -102,6 +102,7 @@ void GlobalStoragePool::restore() data_storage->restore(); meta_storage->restore(); + // We don't need remove `GC task` in getBackgroundPool. gc_handle = global_context.getBackgroundPool().addTask( [this] { return this->gc(global_context.getSettingsRef()); @@ -109,15 +110,6 @@ void GlobalStoragePool::restore() false); } -GlobalStoragePool::~GlobalStoragePool() -{ - if (gc_handle) - { - global_context.getBackgroundPool().removeTask(gc_handle); - gc_handle = nullptr; - } -} - bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { { diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index a4569cafd0c..07c55d0fb70 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -69,8 +69,6 @@ class GlobalStoragePool : private boost::noncopyable GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); - ~GlobalStoragePool(); - void restore(); friend class StoragePool; From 65ed3a11b129f687f2256d1e21173282f8bd9e40 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Mon, 25 Apr 2022 19:38:30 +0800 Subject: [PATCH 08/38] fix tests --- dbms/src/Storages/Page/Page.h | 6 +++--- dbms/src/Storages/Page/PageStorage.h | 13 ++++++------- dbms/src/Storages/Page/V3/PageDirectory.cpp | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dbms/src/Storages/Page/Page.h b/dbms/src/Storages/Page/Page.h index c461f4b7b0b..0217fd82ccf 100644 --- a/dbms/src/Storages/Page/Page.h +++ b/dbms/src/Storages/Page/Page.h @@ -76,16 +76,16 @@ struct Page return ByteBuffer(data.begin() + beg, data.begin() + end); } - inline static PageFieldSizes fieldOffsetsToSizes(std::set & field_offsets, size_t data_size) + inline static PageFieldSizes fieldOffsetsToSizes(const PageFieldOffsetChecksums & field_offsets, size_t data_size) { PageFieldSizes field_size = {}; auto it = field_offsets.begin(); while (it != field_offsets.end()) { - PageFieldOffset beg = it->offset; + PageFieldOffset beg = it->first; ++it; - field_size.emplace_back(it == field_offsets.end() ? data_size - beg : it->offset - beg); + field_size.emplace_back(it == field_offsets.end() ? data_size - beg : it->first - beg); } return field_size; } diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 19c683363be..4bb71e88e4c 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -387,7 +387,7 @@ class PageReader : private boost::noncopyable } } - PageMap read(const std::vector & page_ids) const + PageMap read(const PageIds & page_ids) const { switch (run_mode) { @@ -418,7 +418,7 @@ class PageReader : private boost::noncopyable } } - void read(const std::vector & page_ids, PageHandler & handler) const + void read(const PageIds & page_ids, PageHandler & handler) const { switch (run_mode) { @@ -667,7 +667,7 @@ class PageWriter : private boost::noncopyable case PageStorageRunMode::MIX_MODE: { const auto & ns_id = write_batch.getNamespaceId(); - WriteBatch wb_for_v2{write_batch.getNamespaceId()}; + WriteBatch wb_for_v2{ns_id}; for (const auto & write : write_batch.getWrites()) { switch (write.type) @@ -690,7 +690,6 @@ class PageWriter : private boost::noncopyable write.ori_page_id, /*snapshot*/ nullptr, false); - // if normal id is not ok, read from v2 and create a new put + ref if (resolved_page_id == INVALID_PAGE_ID) { @@ -699,6 +698,7 @@ class PageWriter : private boost::noncopyable { WriteBatch wb_for_put{ns_id}; auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); + assert(entry_for_put.size == page_for_put.data.size()); // Page with fields if (!entry_for_put.field_offsets.empty()) @@ -707,8 +707,7 @@ class PageWriter : private boost::noncopyable 0, std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), page_for_put.data.size(), - Page::fieldOffsetsToSizes(page_for_put.field_offsets, - page_for_put.data.size())); + Page::fieldOffsetsToSizes(entry_for_put.field_offsets, entry_for_put.size)); } else { // Normal page with fields @@ -718,7 +717,7 @@ class PageWriter : private boost::noncopyable page_for_put.data.size()), page_for_put.data.size()); } - storage_v2->write(std::move(wb_for_put), write_limiter); + storage_v3->write(std::move(wb_for_put), write_limiter); } else { diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index aef4e9e1922..e4351ef9e4b 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -831,7 +831,7 @@ PageIdV3Internal PageDirectory::getNormalPageId(PageIdV3Internal page_id, const } else { - return PageIdV3Internal(0, INVALID_PAGE_ID); + return PageIdV3Internal(INVALID_PAGE_ID, 0); } } } @@ -868,7 +868,7 @@ PageIdV3Internal PageDirectory::getNormalPageId(PageIdV3Internal page_id, const } else { - return PageIdV3Internal(0, INVALID_PAGE_ID); + return PageIdV3Internal(INVALID_PAGE_ID, 0); } } From a2f6dd6b52ecdb6e882947af9401b5568f0e885a Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 26 Apr 2022 11:01:24 +0800 Subject: [PATCH 09/38] add tests --- dbms/src/Storages/DeltaMerge/StoragePool.h | 5 + .../Page/V3/tests/gtest_blob_store.cpp | 1 + .../V3/tests/gtest_page_storage_mix_mode.cpp | 432 ++++++++++++++++++ 3 files changed, 438 insertions(+) create mode 100644 dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 07c55d0fb70..6f49412b67d 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -69,6 +69,11 @@ class GlobalStoragePool : private boost::noncopyable GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); + static void destory() + { + global_storage_pool = nullptr; + } + void restore(); friend class StoragePool; diff --git a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp index a141e21ee2a..aa41543bc71 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp @@ -326,6 +326,7 @@ try { for (const auto & [path, stats] : blob_store.blob_stats.getStats()) { + (void)path; for (const auto & stat : stats) { if (stat->id == file_id1) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp new file mode 100644 index 00000000000..1b62fa38830 --- /dev/null +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -0,0 +1,432 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +using namespace DM; + +namespace PS::V3::tests +{ +class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic +{ +public: + void SetUp() override + { + TiFlashStorageTestBasic::SetUp(); + const auto & path = getTemporaryPath(); + createIfNotExist(path); + + auto global_context = DB::tests::TiFlashTestEnv::getContext(); + file_provider = DB::tests::TiFlashTestEnv::getContext().getFileProvider(); + + std::vector caps = {}; + Strings paths = {path}; + + cap_metrics = std::make_shared(0, paths, caps, Strings{}, caps); + storage_path_pool_v2 = std::make_unique(Strings{path}, Strings{path}, "test", "t1", true, cap_metrics, file_provider); + storage_pool_v2 = std::make_unique(PageStorageRunMode::ONLY_V2, TEST_NAMESPACE_ID, nullptr, *storage_path_pool_v2, global_context, "test.t1"); + + storage_path_pool_v3 = std::make_unique(Strings{path}, Strings{path}, Strings{}, cap_metrics, file_provider, true); + GlobalStoragePool::init(*storage_path_pool_v3, global_context, global_context.getSettingsRef()); + storage_pool_mix = std::make_unique(PageStorageRunMode::MIX_MODE, + TEST_NAMESPACE_ID, + GlobalStoragePool::getInstance(), + *storage_path_pool_v2, + global_context, + "test.t1"); + + reloadV2StoragePool(); + } + + void TearDown() override + { + GlobalStoragePool::destory(); + } + + + PageStorageRunMode reloadMixedStoragePool() + { + PageStorageRunMode run_mode = storage_pool_mix->restore(); + page_writer_mix = storage_pool_mix->logWriter(); + page_reader_mix = storage_pool_mix->logReader(); + return run_mode; + } + + void reloadV2StoragePool() + { + storage_pool_v2->restore(); + page_writer_v2 = storage_pool_v2->logWriter(); + page_reader_v2 = storage_pool_v2->logReader(); + } + +protected: + FileProviderPtr file_provider; + PageStorage::Config config; + PathCapacityMetricsPtr cap_metrics; + + std::unique_ptr storage_path_pool_v2; + std::unique_ptr storage_path_pool_v3; + std::unique_ptr storage_pool_v2; + std::unique_ptr storage_pool_mix; + + PageWriterPtr page_writer_v2; + PageWriterPtr page_writer_mix; + + PageReaderPtr page_reader_v2; + PageReaderPtr page_reader_mix; +}; + +inline ::testing::AssertionResult getPageCompare( + const char * /*buff_cmp_expr*/, + const char * buf_size_expr, + const char * /*page_cmp_expr*/, + const char * page_id_expr, + char * buff_cmp, + const size_t buf_size, + const Page & page_cmp, + const PageId & page_id) +{ + if (page_cmp.data.size() != buf_size) + { + return testing::internal::EqFailure( + DB::toString(buf_size).c_str(), + DB::toString(page_cmp.data.size()).c_str(), + buf_size_expr, + "page.data.size()", + false); + } + + if (page_cmp.page_id != page_id) + { + return testing::internal::EqFailure( + DB::toString(page_id).c_str(), + DB::toString(page_cmp.page_id).c_str(), + page_id_expr, + "page.page_id", + false); + } + + if (strncmp(page_cmp.data.begin(), buff_cmp, buf_size) != 0) + { + return ::testing::AssertionFailure( // + ::testing::Message( + "Page data not match the buffer")); + } + + return ::testing::AssertionSuccess(); +} + +#define ASSERT_PAGE_EQ(buff_cmp, buf_size, page_cmp, page_id) \ + ASSERT_PRED_FORMAT4(getPageCompare, buff_cmp, buf_size, page_cmp, page_id) +#define EXPECT_PAGE_EQ(buff_cmp, buf_size, page_cmp, page_id) \ + EXPECT_PRED_FORMAT4(getPageCompare, buff_cmp, buf_size, page_cmp, page_id) + +TEST_F(PageStorageMixedTest, WriteRead) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(2, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + { + const auto & page1 = page_reader_mix->read(1); + const auto & page2 = page_reader_mix->read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page2, 2); + } + + { + WriteBatch batch; + const size_t buf_sz2 = 2048; + char c_buff2[buf_sz2] = {0}; + + ReadBufferPtr buff2 = std::make_shared(c_buff2, sizeof(c_buff2)); + batch.putPage(3, tag, buff2, buf_sz2); + page_writer_mix->write(std::move(batch), nullptr); + + const auto & page3 = page_reader_mix->read(3); + ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3); + reloadV2StoragePool(); + ASSERT_THROW(page_reader_v2->read(3), DB::Exception); + } +} +CATCH + +TEST_F(PageStorageMixedTest, Read) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(2, tag, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + { + const auto & page1 = page_reader_mix->read(1); + const auto & page2 = page_reader_mix->read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page2, 2); + } + + const size_t buf_sz2 = 2048; + char c_buff2[buf_sz2] = {0}; + { + WriteBatch batch; + ReadBufferPtr buff2 = std::make_shared(c_buff2, sizeof(c_buff2)); + batch.putPage(3, tag, buff2, buf_sz2); + buff2 = std::make_shared(c_buff2, sizeof(c_buff2)); + batch.putPage(4, tag, buff2, buf_sz2, {20, 120, 400, 200, 15, 75, 170, 24, 500, 500, 24}); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + PageIds page_ids = {1, 2, 3, 4}; + page_ids.emplace_back(1); + page_ids.emplace_back(2); + page_ids.emplace_back(3); + page_ids.emplace_back(4); + auto page_maps = page_reader_mix->read(page_ids); + ASSERT_EQ(page_maps.size(), 4); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); + ASSERT_PAGE_EQ(c_buff2, buf_sz2, page_maps[3], 3); + ASSERT_PAGE_EQ(c_buff2, buf_sz2, page_maps[4], 4); + } + + { + PageIds page_ids = {1, 2, 3, 4}; + PageHandler hander = [](DB::PageId /*page_id*/, const Page & /*page*/) { + }; + ASSERT_NO_THROW(page_reader_mix->read(page_ids, hander)); + } + + { + std::vector read_fields; + read_fields.emplace_back(std::make_pair(2, {1, 3, 6})); + read_fields.emplace_back(std::make_pair(4, {1, 3, 4, 8, 10})); + PageMap page_maps = page_reader_mix->read(read_fields); + ASSERT_EQ(page_maps.size(), 2); + ASSERT_EQ(page_maps[2].page_id, 2); + ASSERT_EQ(page_maps[2].field_offsets.size(), 3); + ASSERT_EQ(page_maps[4].page_id, 4); + ASSERT_EQ(page_maps[4].field_offsets.size(), 5); + } +} +CATCH + +TEST_F(PageStorageMixedTest, ReadWithSnapshot) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(2, tag, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + const size_t buf_sz2 = 2048; + char c_buff2[buf_sz2] = {0}; + { + WriteBatch batch; + ReadBufferPtr buff2 = std::make_shared(c_buff2, sizeof(c_buff2)); + batch.putPage(3, tag, buff2, buf_sz2); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_mix = page_reader_mix->getSnapshot("ReadWithSnapshotTest"); + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix); + + const auto & page1 = page_reader_mix_with_snap.read(1); + const auto & page2 = page_reader_mix_with_snap.read(2); + const auto & page3 = page_reader_mix_with_snap.read(3); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page2, 2); + ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, true, "ReadWithSnapshotTest"); + const auto & page1 = page_reader_mix_with_snap.read(1); + const auto & page2 = page_reader_mix_with_snap.read(2); + const auto & page3 = page_reader_mix_with_snap.read(3); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page2, 2); + ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3); + } + + { + WriteBatch batch; + ReadBufferPtr buff2 = std::make_shared(c_buff2, sizeof(c_buff2)); + batch.putPage(4, tag, buff2, buf_sz2); + page_writer_mix->write(std::move(batch), nullptr); + } + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix); + ASSERT_THROW(page_reader_mix_with_snap.read(4), DB::Exception); + } +} +CATCH + + +TEST_F(PageStorageMixedTest, PutExt) +try +{ + { + WriteBatch batch; + batch.putExternal(1, 0); + batch.putExternal(2, 0); + batch.putExternal(3, 0); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); +} +CATCH + + +TEST_F(PageStorageMixedTest, Del) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + batch.delPage(1); + ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); + } + + reloadV2StoragePool(); + ASSERT_THROW(page_reader_v2->read(1), DB::Exception); +} +CATCH + +TEST_F(PageStorageMixedTest, Ref) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(2, 0, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + const auto & entry = page_reader_mix->getPageEntry(2); + ASSERT_EQ(entry.field_offsets.size(), 8); + } + + { + WriteBatch batch; + batch.putRefPage(3, 1); + ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); + ASSERT_EQ(page_reader_mix->getNormalPageId(3), 1); + } + + reloadV2StoragePool(); + ASSERT_THROW(page_reader_v2->read(3), DB::Exception); + + { + WriteBatch batch; + batch.putRefPage(4, 2); + + ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); + ASSERT_EQ(page_reader_mix->getNormalPageId(4), 2); + + std::vector read_fields; + read_fields.emplace_back(std::make_pair(4, {0, 1, 2, 6})); + + PageMap page_maps = page_reader_mix->read(read_fields); + ASSERT_EQ(page_maps.size(), 1); + ASSERT_EQ(page_maps[4].page_id, 4); + ASSERT_EQ(page_maps[4].field_offsets.size(), 4); + ASSERT_EQ(page_maps[4].data.size(), 710); + + auto field_offset = page_maps[4].field_offsets; + auto it = field_offset.begin(); + ASSERT_EQ(it->offset, 0); + ++it; + ASSERT_EQ(it->offset, 20); + ++it; + ASSERT_EQ(it->offset, 140); + ++it; + ASSERT_EQ(it->offset, 540); + } +} +CATCH + +} // namespace PS::V3::tests +} // namespace DB From 0be35b7d707fe26c605eec60e558196bd0902b5f Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 26 Apr 2022 15:04:38 +0800 Subject: [PATCH 10/38] change back globalpathpool --- dbms/src/Interpreters/Context.cpp | 31 ++++++++++++++++++ dbms/src/Interpreters/Context.h | 4 +++ dbms/src/Server/DTTool/DTToolBench.cpp | 2 +- dbms/src/Server/tests/gtest_dttool.cpp | 2 +- dbms/src/Server/tests/gtest_server_config.cpp | 2 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 11 ++----- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 14 ++++++-- dbms/src/Storages/DeltaMerge/StoragePool.h | 32 ++----------------- .../tests/gtest_dm_delta_value_space.cpp | 2 +- .../DeltaMerge/tests/gtest_dm_file.cpp | 4 +-- .../DeltaMerge/tests/gtest_dm_segment.cpp | 2 +- .../tests/gtest_dm_segment_common_handle.cpp | 2 +- .../V3/tests/gtest_page_storage_mix_mode.cpp | 17 ++++------ dbms/src/TestUtils/TiFlashTestEnv.cpp | 1 + 14 files changed, 67 insertions(+), 59 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 627a84c9398..a7ff274a270 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -162,6 +162,7 @@ struct ContextShared FileProviderPtr file_provider; /// File provider. IORateLimiter io_rate_limiter; PageStorageRunMode storage_run_mode; + DM::GlobalStoragePoolPtr global_storage_pool; /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. class SessionKeyHash @@ -1591,6 +1592,36 @@ void Context::setPageStorageRunMode(PageStorageRunMode run_mode) const shared->storage_run_mode = run_mode; } +bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool) +{ + auto lock = getLock(); + if (shared->storage_run_mode == PageStorageRunMode::MIX_MODE || shared->storage_run_mode == PageStorageRunMode::ONLY_V3) + { + try + { + shared->global_storage_pool = std::make_shared(path_pool, *this, settings); + shared->global_storage_pool->restore(); + return true; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + throw; + } + } + else + { + shared->global_storage_pool = nullptr; + return false; + } +} + +DM::GlobalStoragePoolPtr Context::getGlobalStoragePool() const +{ + auto lock = getLock(); + return shared->global_storage_pool; +} + UInt16 Context::getTCPPort() const { auto lock = getLock(); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 93435a69ba5..922a9047013 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -104,6 +104,8 @@ namespace DM { class MinMaxIndexCache; class DeltaIndexManager; +class GlobalStoragePool; +using GlobalStoragePoolPtr = std::shared_ptr; } // namespace DM /// (database name, table name) @@ -407,6 +409,8 @@ class Context void initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3); void setPageStorageRunMode(PageStorageRunMode run_mode) const; PageStorageRunMode getPageStorageRunMode() const; + bool initializeGlobalStoragePoolIfNeed(const PathPool & path_pool); + DM::GlobalStoragePoolPtr getGlobalStoragePool() const; /// Call after initialization before using system logs. Call for global context. void initializeSystemLogs(); diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index 3639a29d712..e30f95e14b2 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -336,7 +336,7 @@ int benchEntry(const std::vector & opts) auto settings = DB::Settings(); auto db_context = env.getContext(); auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); + auto storage_pool = std::make_unique(*db_context, /*ns_id*/ 1, *path_pool, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index 6d1a550d5b1..a03fc779ae3 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -72,7 +72,7 @@ struct DTToolTest : public DB::base::TiFlashStorageTestBasic properties.push_back(property); } auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - auto storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 1, nullptr, *path_pool, *db_context, "test.t1"); + auto storage_pool = std::make_unique(*db_context, /*ns_id*/ 1, *path_pool, "test.t1"); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index e3ba77949af..a90be5be130 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -362,7 +362,7 @@ dt_page_gc_low_write_prob = 0.2 auto & global_ctx = TiFlashTestEnv::getGlobalContext(); std::unique_ptr path_pool = std::make_unique(global_ctx.getPathPool().withTable("test", "t1", false)); - std::unique_ptr storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, global_ctx, "test.t1"); + std::unique_ptr storage_pool = std::make_unique(global_ctx, /*ns_id*/ 100, *path_pool, "test.t1"); auto verify_storage_pool_reload_config = [&global_ctx](std::unique_ptr & storage_pool) { DB::Settings & settings = global_ctx.getSettingsRef(); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index d7bfc6427b7..f0cd37e39c2 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -210,22 +210,14 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, , hash_salt(++DELTA_MERGE_STORE_HASH_SALT) , log(Logger::get("DeltaMergeStore", fmt::format("{}.{}", db_name, table_name))) { - auto page_storage_run_mode = global_context.getPageStorageRunMode(); LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}]", db_name, table_name); // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; - if (page_storage_run_mode == PageStorageRunMode::ONLY_V3 || page_storage_run_mode == PageStorageRunMode::MIX_MODE) - { - GlobalStoragePool::init(global_context.getPathPool(), global_context, db_context.getSettingsRef()); - } - - storage_pool = std::make_shared(page_storage_run_mode, + storage_pool = std::make_shared(global_context, ns_id, - GlobalStoragePool::getInstance(), path_pool, - global_context, db_name_ + "." + table_name_); // Restore existing dm files and set capacity for path_pool. @@ -245,6 +237,7 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, store_columns = generateStoreColumns(original_table_columns, is_common_handle); auto dm_context = newDMContext(db_context, db_context.getSettingsRef()); + PageStorageRunMode page_storage_run_mode; try { page_storage_run_mode = storage_pool->restore(); // restore from disk diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index da145e86ce0..f6a56de46a4 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -96,6 +96,15 @@ GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & globa , global_context(global_ctx) {} +GlobalStoragePool::~GlobalStoragePool() +{ + if (gc_handle) + { + global_context.getBackgroundPool().removeTask(gc_handle); + gc_handle = nullptr; + } +} + void GlobalStoragePool::restore() { log_storage->restore(); @@ -140,11 +149,12 @@ bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_per return done_anything; } -StoragePool::StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & storage_path_pool, Context & global_ctx, const String & name) - : run_mode(mode) +StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPool & storage_path_pool, const String & name) + : run_mode(global_ctx.getPageStorageRunMode()) , ns_id(ns_id_) , global_context(global_ctx) { + const auto & global_storage_pool = global_context.getGlobalStoragePool(); switch (run_mode) { case PageStorageRunMode::ONLY_V2: diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 6f49412b67d..ea516438b50 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -42,37 +42,9 @@ class GlobalStoragePool : private boost::noncopyable using Timepoint = Clock::time_point; using Seconds = std::chrono::seconds; - // not thread safe - static void init(const PathPool & path_pool, Context & global_ctx, const Settings & settings) - { - if (global_storage_pool != nullptr) - { - return; - } - - try - { - global_storage_pool = std::make_shared(path_pool, global_ctx, settings); - global_storage_pool->restore(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - throw; - } - } - - static std::shared_ptr getInstance() - { - return global_storage_pool; - } - GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); - static void destory() - { - global_storage_pool = nullptr; - } + ~GlobalStoragePool(); void restore(); @@ -104,7 +76,7 @@ class StoragePool : private boost::noncopyable using Timepoint = Clock::time_point; using Seconds = std::chrono::seconds; - StoragePool(PageStorageRunMode mode, NamespaceId ns_id_, const GlobalStoragePoolPtr & global_storage_pool, StoragePathPool & path_pool, Context & global_ctx, const String & name = ""); + StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPool & path_pool, const String & name = ""); PageStorageRunMode restore(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp index 57abe397e63..2b64fd90c09 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp @@ -84,7 +84,7 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, table_id, nullptr, *storage_path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(*db_context, table_id, *storage_path_pool, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index a58e977926e..3ddf318509f 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -105,7 +105,7 @@ class DMFile_Test parent_path = TiFlashStorageTestBasic::getTemporaryPath(); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFile_Test", false)); - storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(*db_context, /*ns_id*/ 100, *path_pool, "test.t1"); dm_file = DMFile::create(1, parent_path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); @@ -1057,7 +1057,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic auto configuration = mode == DMFileMode::DirectoryChecksum ? std::make_optional() : std::nullopt; path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, table_id, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(*db_context, table_id, *path_pool, "test.t1"); dm_file = DMFile::create(0, path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index e7f73386bb8..6f1da767923 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -74,7 +74,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); - storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*ns_id*/ 100, nullptr, *storage_path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(*db_context, /*ns_id*/ 100, *storage_path_pool, "test.t1"); storage_pool->restore(); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index e2d6ef0d20c..1cc61663a2f 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -53,7 +53,7 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic { TiFlashStorageTestBasic::reload(std::move(db_settings)); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); - storage_pool = std::make_unique(PageStorageRunMode::ONLY_V2, /*table_id*/ 100, nullptr, *path_pool, *db_context, "test.t1"); + storage_pool = std::make_unique(*db_context, /*table_id*/ 100, *path_pool, "test.t1"); storage_pool->restore(); if (!cols) cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 1b62fa38830..8ac15ca4747 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -43,28 +43,24 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic cap_metrics = std::make_shared(0, paths, caps, Strings{}, caps); storage_path_pool_v2 = std::make_unique(Strings{path}, Strings{path}, "test", "t1", true, cap_metrics, file_provider); - storage_pool_v2 = std::make_unique(PageStorageRunMode::ONLY_V2, TEST_NAMESPACE_ID, nullptr, *storage_path_pool_v2, global_context, "test.t1"); + global_context.setPageStorageRunMode(PageStorageRunMode::ONLY_V2); + storage_pool_v2 = std::make_unique(global_context, TEST_NAMESPACE_ID, *storage_path_pool_v2, "test.t1"); storage_path_pool_v3 = std::make_unique(Strings{path}, Strings{path}, Strings{}, cap_metrics, file_provider, true); - GlobalStoragePool::init(*storage_path_pool_v3, global_context, global_context.getSettingsRef()); - storage_pool_mix = std::make_unique(PageStorageRunMode::MIX_MODE, + global_context.setPageStorageRunMode(PageStorageRunMode::MIX_MODE); + global_context.initializeGlobalStoragePoolIfNeed(*storage_path_pool_v3); + storage_pool_mix = std::make_unique(global_context, TEST_NAMESPACE_ID, - GlobalStoragePool::getInstance(), *storage_path_pool_v2, - global_context, "test.t1"); reloadV2StoragePool(); } - void TearDown() override - { - GlobalStoragePool::destory(); - } - PageStorageRunMode reloadMixedStoragePool() { + DB::tests::TiFlashTestEnv::getContext().setPageStorageRunMode(PageStorageRunMode::MIX_MODE); PageStorageRunMode run_mode = storage_pool_mix->restore(); page_writer_mix = storage_pool_mix->logWriter(); page_reader_mix = storage_pool_mix->logReader(); @@ -73,6 +69,7 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic void reloadV2StoragePool() { + DB::tests::TiFlashTestEnv::getContext().setPageStorageRunMode(PageStorageRunMode::ONLY_V2); storage_pool_v2->restore(); page_writer_v2 = storage_pool_v2->logWriter(); page_reader_v2 = storage_pool_v2->logReader(); diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index f7b1ba9b56c..919d1d9fde0 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -58,6 +58,7 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ testdata_path.swap(absolute_testdata_path); } global_context->initializePathCapacityMetric(0, testdata_path, {}, {}, {}); + global_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2); auto paths = getPathPool(testdata_path); global_context->setPathPool( From a2819bc645056ba4b7ecfd033bdaebab4068d306 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 27 Apr 2022 18:20:59 +0800 Subject: [PATCH 11/38] fix ut problem --- dbms/src/Interpreters/Context.cpp | 9 ++ dbms/src/Server/Server.cpp | 1 + dbms/src/Storages/DeltaMerge/StoragePool.cpp | 9 -- dbms/src/Storages/DeltaMerge/StoragePool.h | 2 - dbms/src/Storages/Page/PageStorage.h | 4 + .../V3/tests/gtest_page_storage_mix_mode.cpp | 97 ++++++++++++++----- dbms/src/TestUtils/TiFlashTestEnv.cpp | 9 +- 7 files changed, 90 insertions(+), 41 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index a7ff274a270..f719de50cb9 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1595,6 +1595,15 @@ void Context::setPageStorageRunMode(PageStorageRunMode run_mode) const bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool) { auto lock = getLock(); + std::cout << "path_pool_v3 addr2 : " << &path_pool << std::endl; + if (shared->global_storage_pool) + { + // Can't init GlobalStoragePool twice. + // Because we won't remove the gc task in BackGroundPool + // Also won't remove it from ~GlobalStoragePool() + throw Exception("Can't initializeGlobalStoragePool twice", ErrorCodes::LOGICAL_ERROR); + } + if (shared->storage_run_mode == PageStorageRunMode::MIX_MODE || shared->storage_run_mode == PageStorageRunMode::ONLY_V3) { try diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index f5689797c37..1b3a8113e67 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1114,6 +1114,7 @@ int Server::main(const std::vector & /*args*/) global_context->getFileProvider()); global_context->initializePageStorageMode(global_context->getPathPool(), storage_config.enable_ps_v3); + global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); // Use pd address to define which default_database we use by default. // For mock test, we use "default". For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index f6a56de46a4..a850e994f89 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -94,15 +94,7 @@ GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & globa global_ctx.getFileProvider(), true)) , global_context(global_ctx) -{} - -GlobalStoragePool::~GlobalStoragePool() { - if (gc_handle) - { - global_context.getBackgroundPool().removeTask(gc_handle); - gc_handle = nullptr; - } } void GlobalStoragePool::restore() @@ -111,7 +103,6 @@ void GlobalStoragePool::restore() data_storage->restore(); meta_storage->restore(); - // We don't need remove `GC task` in getBackgroundPool. gc_handle = global_context.getBackgroundPool().addTask( [this] { return this->gc(global_context.getSettingsRef()); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index ea516438b50..b87cf354474 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -44,8 +44,6 @@ class GlobalStoragePool : private boost::noncopyable GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); - ~GlobalStoragePool(); - void restore(); friend class StoragePool; diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 4bb71e88e4c..9568d539f54 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -34,6 +36,7 @@ #include #include + namespace DB { class FileProvider; @@ -718,6 +721,7 @@ class PageWriter : private boost::noncopyable page_for_put.data.size()); } storage_v3->write(std::move(wb_for_put), write_limiter); + LOG_FMT_INFO(Logger::get("PageWriter"), "Can't find [origin_id={}] in v3. Created a new page with [field_offsets={}] into V3", write.ori_page_id, entry_for_put.field_offsets.size()); } else { diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 8ac15ca4747..16b8d381624 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -23,12 +23,30 @@ namespace DB { using namespace DM; - +using namespace tests; namespace PS::V3::tests { class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic { public: + static void SetUpTestCase() + { + auto path = TiFlashTestEnv::getTemporaryPath("PageStorageMixedTestV3Path"); + dropDataOnDisk(path); + createIfNotExist(path); + + std::vector caps = {}; + Strings paths = {path}; + + auto global_context = TiFlashTestEnv::getContext(); + + storage_path_pool_v3 = std::make_unique(Strings{path}, Strings{path}, Strings{}, std::make_shared(0, paths, caps, Strings{}, caps), global_context.getFileProvider(), true); + + global_context.setPageStorageRunMode(PageStorageRunMode::MIX_MODE); + if (!global_context.getGlobalStoragePool()) + global_context.initializeGlobalStoragePoolIfNeed(*storage_path_pool_v3); + } + void SetUp() override { TiFlashStorageTestBasic::SetUp(); @@ -36,19 +54,17 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic createIfNotExist(path); auto global_context = DB::tests::TiFlashTestEnv::getContext(); - file_provider = DB::tests::TiFlashTestEnv::getContext().getFileProvider(); std::vector caps = {}; Strings paths = {path}; - cap_metrics = std::make_shared(0, paths, caps, Strings{}, caps); - storage_path_pool_v2 = std::make_unique(Strings{path}, Strings{path}, "test", "t1", true, cap_metrics, file_provider); + PathCapacityMetricsPtr cap_metrics = std::make_shared(0, paths, caps, Strings{}, caps); + storage_path_pool_v2 = std::make_unique(Strings{path}, Strings{path}, "test", "t1", true, cap_metrics, global_context.getFileProvider()); + global_context.setPageStorageRunMode(PageStorageRunMode::ONLY_V2); storage_pool_v2 = std::make_unique(global_context, TEST_NAMESPACE_ID, *storage_path_pool_v2, "test.t1"); - storage_path_pool_v3 = std::make_unique(Strings{path}, Strings{path}, Strings{}, cap_metrics, file_provider, true); global_context.setPageStorageRunMode(PageStorageRunMode::MIX_MODE); - global_context.initializeGlobalStoragePoolIfNeed(*storage_path_pool_v3); storage_pool_mix = std::make_unique(global_context, TEST_NAMESPACE_ID, *storage_path_pool_v2, @@ -57,7 +73,6 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic reloadV2StoragePool(); } - PageStorageRunMode reloadMixedStoragePool() { DB::tests::TiFlashTestEnv::getContext().setPageStorageRunMode(PageStorageRunMode::MIX_MODE); @@ -76,12 +91,8 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic } protected: - FileProviderPtr file_provider; - PageStorage::Config config; - PathCapacityMetricsPtr cap_metrics; - std::unique_ptr storage_path_pool_v2; - std::unique_ptr storage_path_pool_v3; + static std::unique_ptr storage_path_pool_v3; std::unique_ptr storage_pool_v2; std::unique_ptr storage_pool_mix; @@ -92,6 +103,8 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic PageReaderPtr page_reader_mix; }; +std::unique_ptr PageStorageMixedTest::storage_path_pool_v3 = nullptr; + inline ::testing::AssertionResult getPageCompare( const char * /*buff_cmp_expr*/, const char * buf_size_expr, @@ -179,6 +192,13 @@ try reloadV2StoragePool(); ASSERT_THROW(page_reader_v2->read(3), DB::Exception); } + + { + // Revert v3 + WriteBatch batch; + batch.delPage(3); + page_writer_mix->write(std::move(batch), nullptr); + } } CATCH @@ -253,6 +273,14 @@ try ASSERT_EQ(page_maps[4].page_id, 4); ASSERT_EQ(page_maps[4].field_offsets.size(), 5); } + + { + // Revert v3 + WriteBatch batch; + batch.delPage(3); + batch.delPage(4); + page_writer_mix->write(std::move(batch), nullptr); + } } CATCH @@ -319,6 +347,14 @@ try auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix); ASSERT_THROW(page_reader_mix_with_snap.read(4), DB::Exception); } + + { + // Revert v3 + WriteBatch batch; + batch.delPage(3); + batch.delPage(4); + page_writer_mix->write(std::move(batch), nullptr); + } } CATCH @@ -374,45 +410,46 @@ try { WriteBatch batch; ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); - batch.putPage(1, 0, buff, buf_sz); + batch.putPage(7, 0, buff, buf_sz); buff = std::make_shared(c_buff, sizeof(c_buff)); - batch.putPage(2, 0, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); + batch.putPage(8, 0, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); page_writer_v2->write(std::move(batch), nullptr); } { ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); - const auto & entry = page_reader_mix->getPageEntry(2); + const auto & entry = page_reader_mix->getPageEntry(8); ASSERT_EQ(entry.field_offsets.size(), 8); } { WriteBatch batch; - batch.putRefPage(3, 1); - ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); - ASSERT_EQ(page_reader_mix->getNormalPageId(3), 1); + batch.putRefPage(9, 7); + // ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); + page_writer_mix->write(std::move(batch), nullptr); + ASSERT_EQ(page_reader_mix->getNormalPageId(9), 7); } reloadV2StoragePool(); - ASSERT_THROW(page_reader_v2->read(3), DB::Exception); + ASSERT_THROW(page_reader_v2->read(9), DB::Exception); { WriteBatch batch; - batch.putRefPage(4, 2); + batch.putRefPage(10, 8); ASSERT_NO_THROW(page_writer_mix->write(std::move(batch), nullptr)); - ASSERT_EQ(page_reader_mix->getNormalPageId(4), 2); + ASSERT_EQ(page_reader_mix->getNormalPageId(10), 8); std::vector read_fields; - read_fields.emplace_back(std::make_pair(4, {0, 1, 2, 6})); + read_fields.emplace_back(std::make_pair(10, {0, 1, 2, 6})); PageMap page_maps = page_reader_mix->read(read_fields); ASSERT_EQ(page_maps.size(), 1); - ASSERT_EQ(page_maps[4].page_id, 4); - ASSERT_EQ(page_maps[4].field_offsets.size(), 4); - ASSERT_EQ(page_maps[4].data.size(), 710); + ASSERT_EQ(page_maps[10].page_id, 10); + ASSERT_EQ(page_maps[10].field_offsets.size(), 4); + ASSERT_EQ(page_maps[10].data.size(), 710); - auto field_offset = page_maps[4].field_offsets; + auto field_offset = page_maps[10].field_offsets; auto it = field_offset.begin(); ASSERT_EQ(it->offset, 0); ++it; @@ -422,6 +459,14 @@ try ++it; ASSERT_EQ(it->offset, 540); } + + { + // Revert v3 + WriteBatch batch; + batch.delPage(9); + batch.delPage(10); + page_writer_mix->write(std::move(batch), nullptr); + } } CATCH diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index 919d1d9fde0..ebcbbc8179a 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -68,6 +68,11 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ true, global_context->getPathCapacity(), global_context->getFileProvider()); + + global_context->setPageStorageRunMode(enable_ps_v3 ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::ONLY_V2); + global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); + LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); + TiFlashRaftConfig raft_config; raft_config.ignore_databases = {"default", "system"}; @@ -75,10 +80,6 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ raft_config.disable_bg_flush = true; global_context->createTMTContext(raft_config, pingcap::ClusterConfig()); - - global_context->setPageStorageRunMode(enable_ps_v3 ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::ONLY_V2); - LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); - global_context->setDeltaIndexManager(1024 * 1024 * 100 /*100MB*/); global_context->getTMTContext().restore(); From d921d0271283089ae0f982da40b3d876f20fa115 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 27 Apr 2022 20:49:03 +0800 Subject: [PATCH 12/38] update --- dbms/src/Interpreters/Context.cpp | 1 - dbms/src/Storages/DeltaMerge/StoragePool.cpp | 13 +++++-------- dbms/src/Storages/DeltaMerge/StoragePool.h | 2 -- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index f719de50cb9..67a0620d2f1 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1595,7 +1595,6 @@ void Context::setPageStorageRunMode(PageStorageRunMode run_mode) const bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool) { auto lock = getLock(); - std::cout << "path_pool_v3 addr2 : " << &path_pool << std::endl; if (shared->global_storage_pool) { // Can't init GlobalStoragePool twice. diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index a850e994f89..806c2f55833 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -112,15 +112,12 @@ void GlobalStoragePool::restore() bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { - { - std::lock_guard lock(mutex); - - Timepoint now = Clock::now(); - if (now < (last_try_gc_time.load() + try_gc_period)) - return false; + // No need lock + Timepoint now = Clock::now(); + if (now < (last_try_gc_time.load() + try_gc_period)) + return false; - last_try_gc_time = now; - } + last_try_gc_time = now; bool done_anything = false; auto write_limiter = global_context.getWriteLimiter(); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index b87cf354474..4aa09c65925 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -60,8 +60,6 @@ class GlobalStoragePool : private boost::noncopyable std::atomic last_try_gc_time = Clock::now(); - std::mutex mutex; - Context & global_context; BackgroundProcessingPool::TaskHandle gc_handle; }; From 89b2b26ce4ab0d55f0f6f5a0a37d2fe8d8b22a71 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 27 Apr 2022 21:32:28 +0800 Subject: [PATCH 13/38] update --- dbms/src/Interpreters/Context.cpp | 1 + dbms/src/Server/Server.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 67a0620d2f1..45906746110 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1575,6 +1575,7 @@ void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ if (!enable_ps_v3) { shared->storage_run_mode = PageStorageRunMode::ONLY_V2; + return; } shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 1b3a8113e67..ecd69805e97 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1115,6 +1115,7 @@ int Server::main(const std::vector & /*args*/) global_context->initializePageStorageMode(global_context->getPathPool(), storage_config.enable_ps_v3); global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); + LOG_FMT_INFO(log, "Global PageStorage run mode is {}", static_cast(global_context->getPageStorageRunMode())); // Use pd address to define which default_database we use by default. // For mock test, we use "default". For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. From de9ccd8062f5e177a2469ccbe5b5f944cabf40a7 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 29 Apr 2022 00:34:04 +0800 Subject: [PATCH 14/38] update --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 10 +++ dbms/src/Storages/DeltaMerge/StoragePool.h | 2 + dbms/src/Storages/Page/PageStorage.h | 72 ++++++++++++++----- .../V3/tests/gtest_page_storage_mix_mode.cpp | 11 ++- dbms/src/Storages/Page/WriteBatch.h | 8 +++ 5 files changed, 81 insertions(+), 22 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 806c2f55833..353b2331ba0 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -97,6 +97,16 @@ GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & globa { } + +GlobalStoragePool::~GlobalStoragePool() +{ + if (gc_handle) + { + global_context.getBackgroundPool().removeTask(gc_handle); + gc_handle = nullptr; + } +} + void GlobalStoragePool::restore() { log_storage->restore(); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 4aa09c65925..603e9dc495b 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -44,6 +44,8 @@ class GlobalStoragePool : private boost::noncopyable GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); + ~GlobalStoragePool(); + void restore(); friend class StoragePool; diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 9568d539f54..cd9e4eb7604 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -405,15 +405,25 @@ class PageReader : private boost::noncopyable case PageStorageRunMode::MIX_MODE: { auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteV3Snapshot(), false); - auto it = page_maps.begin(); - while (it != page_maps.end()) + PageIds invalid_page_ids; + for (const auto & [query_page_id, page] : page_maps) { - if (it->second.page_id == INVALID_PAGE_ID) + // TODO : change to isvalid(), after pre pr merged. + if (page.page_id == INVALID_PAGE_ID) { - it->second = storage_v2->read(ns_id, it->first, read_limiter, toConcreteV2Snapshot()); + invalid_page_ids.emplace_back(query_page_id); } - it++; } + + if (!invalid_page_ids.empty()) + { + const auto & page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_ids, read_limiter, toConcreteV2Snapshot()); + for (const auto & [page_id_, page_] : page_maps_from_v2) + { + page_maps[page_id_] = page_; + } + } + return page_maps; } default: @@ -463,11 +473,23 @@ class PageReader : private boost::noncopyable { auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteV3Snapshot(), false); + std::vector invalid_page_fields; + for (const auto & page_field : page_fields) { + // TODO : change to isvalid(), after pre pr merged. if (page_maps[page_field.first].page_id == INVALID_PAGE_ID) { - page_maps[page_field.first] = storage_v2->read(ns_id, page_field, read_limiter, toConcreteV2Snapshot()); + invalid_page_fields.emplace_back(page_field); + } + } + + if (!invalid_page_fields.empty()) + { + auto page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_fields, read_limiter, toConcreteV2Snapshot()); + for (const auto & page_field_ : invalid_page_fields) + { + page_maps[page_field_.first] = page_maps_from_v2[page_field_.first]; } } @@ -671,6 +693,8 @@ class PageWriter : private boost::noncopyable { const auto & ns_id = write_batch.getNamespaceId(); WriteBatch wb_for_v2{ns_id}; + WriteBatch wb_for_put_v3{ns_id}; + for (const auto & write : write_batch.getWrites()) { switch (write.type) @@ -699,28 +723,27 @@ class PageWriter : private boost::noncopyable const auto & entry_for_put = storage_v2->getEntry(ns_id, write.ori_page_id, /*snapshot*/ {}); if (entry_for_put.file_id != 0) { - WriteBatch wb_for_put{ns_id}; auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); assert(entry_for_put.size == page_for_put.data.size()); // Page with fields if (!entry_for_put.field_offsets.empty()) { - wb_for_put.putPage(write.ori_page_id, // - 0, - std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), - page_for_put.data.size(), - Page::fieldOffsetsToSizes(entry_for_put.field_offsets, entry_for_put.size)); + wb_for_put_v3.putPage(write.ori_page_id, // + 0, + std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), + page_for_put.data.size(), + Page::fieldOffsetsToSizes(entry_for_put.field_offsets, entry_for_put.size)); } else { // Normal page with fields - wb_for_put.putPage(write.ori_page_id, // - 0, - std::make_shared(page_for_put.data.begin(), - page_for_put.data.size()), - page_for_put.data.size()); + wb_for_put_v3.putPage(write.ori_page_id, // + 0, + std::make_shared(page_for_put.data.begin(), + page_for_put.data.size()), + page_for_put.data.size()); } - storage_v3->write(std::move(wb_for_put), write_limiter); + LOG_FMT_INFO(Logger::get("PageWriter"), "Can't find [origin_id={}] in v3. Created a new page with [field_offsets={}] into V3", write.ori_page_id, entry_for_put.field_offsets.size()); } else @@ -742,7 +765,18 @@ class PageWriter : private boost::noncopyable } } - storage_v3->write(std::move(write_batch), write_limiter); + if (!wb_for_put_v3.empty()) + { + // The `writes` in wb_for_put_v3 must come before the `writes` in write_batch + wb_for_put_v3.copyWrites(write_batch.getWrites()); + storage_v3->write(std::move(wb_for_put_v3), write_limiter); + } + else + { + storage_v3->write(std::move(write_batch), write_limiter); + } + + if (!wb_for_v2.empty()) { storage_v2->write(std::move(wb_for_v2), write_limiter); diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 16b8d381624..d1b8f161679 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -38,7 +38,7 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic std::vector caps = {}; Strings paths = {path}; - auto global_context = TiFlashTestEnv::getContext(); + auto & global_context = TiFlashTestEnv::getGlobalContext(); storage_path_pool_v3 = std::make_unique(Strings{path}, Strings{path}, Strings{}, std::make_shared(0, paths, caps, Strings{}, caps), global_context.getFileProvider(), true); @@ -53,7 +53,7 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic const auto & path = getTemporaryPath(); createIfNotExist(path); - auto global_context = DB::tests::TiFlashTestEnv::getContext(); + auto & global_context = DB::tests::TiFlashTestEnv::getGlobalContext(); std::vector caps = {}; Strings paths = {path}; @@ -219,6 +219,8 @@ try batch.putPage(1, tag, buff, buf_sz); buff = std::make_shared(c_buff, sizeof(c_buff)); batch.putPage(2, tag, buff, buf_sz, {20, 120, 400, 200, 15, 75, 170, 24}); + buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(7, tag, buff, buf_sz, {500, 500, 24}); page_writer_v2->write(std::move(batch), nullptr); } @@ -266,12 +268,15 @@ try std::vector read_fields; read_fields.emplace_back(std::make_pair(2, {1, 3, 6})); read_fields.emplace_back(std::make_pair(4, {1, 3, 4, 8, 10})); + read_fields.emplace_back(std::make_pair(7, {0, 1, 2})); PageMap page_maps = page_reader_mix->read(read_fields); - ASSERT_EQ(page_maps.size(), 2); + ASSERT_EQ(page_maps.size(), 3); ASSERT_EQ(page_maps[2].page_id, 2); ASSERT_EQ(page_maps[2].field_offsets.size(), 3); ASSERT_EQ(page_maps[4].page_id, 4); ASSERT_EQ(page_maps[4].field_offsets.size(), 5); + ASSERT_EQ(page_maps[7].page_id, 7); + ASSERT_EQ(page_maps[7].field_offsets.size(), 3); } { diff --git a/dbms/src/Storages/Page/WriteBatch.h b/dbms/src/Storages/Page/WriteBatch.h index 419f88e08ed..30180874fe6 100644 --- a/dbms/src/Storages/Page/WriteBatch.h +++ b/dbms/src/Storages/Page/WriteBatch.h @@ -189,6 +189,14 @@ class WriteBatch : private boost::noncopyable writes.emplace_back(write); } + void copyWrites(const Writes & writes_) + { + for (const auto & write_ : writes_) + { + copyWrite(write_); + } + } + void clear() { Writes tmp; From 21fd08a32b6d04137141243183a5fc9189985ed5 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 29 Apr 2022 10:44:37 +0800 Subject: [PATCH 15/38] sep gc method --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 37 ++++++++++++-------- dbms/src/Storages/DeltaMerge/StoragePool.h | 5 ++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 353b2331ba0..8ed371c20c6 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -308,6 +308,27 @@ void StoragePool::dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) data_storage_v2->unregisterExternalPagesCallbacks(ns_id); } + +bool StoragePool::doV2Gc(const Settings & settings) +{ + bool done_anything = false; + auto write_limiter = global_context.getWriteLimiter(); + auto read_limiter = global_context.getReadLimiter(); + + auto config = extractConfig(settings, StorageType::Meta); + meta_storage_v2->reloadSettings(config); + done_anything |= meta_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Data); + data_storage_v2->reloadSettings(config); + done_anything |= data_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Log); + log_storage_v2->reloadSettings(config); + done_anything |= log_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); + return done_anything; +} + bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { if (run_mode == PageStorageRunMode::ONLY_V3) @@ -325,21 +346,7 @@ bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) } // Only do the v2 GC - bool done_anything = false; - auto write_limiter = global_context.getWriteLimiter(); - auto read_limiter = global_context.getReadLimiter(); - auto config = extractConfig(settings, StorageType::Meta); - meta_storage_v2->reloadSettings(config); - done_anything |= meta_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Data); - data_storage_v2->reloadSettings(config); - done_anything |= data_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Log); - log_storage_v2->reloadSettings(config); - done_anything |= log_storage_v2->gc(/*not_skip*/ false, write_limiter, read_limiter); - return done_anything; + return doV2Gc(settings); } void StoragePool::shutdown() diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 603e9dc495b..7895da81efb 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -151,7 +151,10 @@ class StoragePool : private boost::noncopyable PageId maxMetaPageId() { return max_meta_page_id; } PageId newLogPageId() { return ++max_log_page_id; } PageId newMetaPageId() { return ++max_meta_page_id; } - +#ifndef DBMS_PUBLIC_GTEST +private: +#endif + bool doV2Gc(const Settings & settings); #ifndef DBMS_PUBLIC_GTEST private: #endif From 1a27a96cbaf1f31a6c2bcdee8ace12fc47df6476 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 5 May 2022 14:52:02 +0800 Subject: [PATCH 16/38] fix --- dbms/src/Interpreters/Context.cpp | 11 +++++++---- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 9 ++++++--- dbms/src/Storages/DeltaMerge/StoragePool.h | 3 +++ dbms/src/TestUtils/TiFlashTestEnv.cpp | 1 - 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 45906746110..7434eca0e61 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1548,10 +1548,9 @@ ReadLimiterPtr Context::getReadLimiter() const return getIORateLimiter().getReadLimiter(); } -static bool isPageStorageV2Existed(const PathPool & path_pool) +static bool isPageStorageV2NotExisted(const PathPool & path_pool) { // Check whether there are any files in kvstore path, if exists, then this is not a new node. - // If it's a new node, then we enable v3. Otherwise, we use v2. for (const auto & path : path_pool.listKVStorePaths()) { Poco::File dir(path); @@ -1572,13 +1571,17 @@ void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ { auto lock = getLock(); + // If it's a new node also enable_ps_v3 is 1, then we direct use ONLY_V3. + // If it's not a new node also enable_ps_v3 is 1, then we use MIX_MODE. + // If it's not a new node also enable_ps_v3 is 0, then we use Only_V2. + // If it's a new node also enable_ps_v3 is 0, then we use Only_V2. if (!enable_ps_v3) { shared->storage_run_mode = PageStorageRunMode::ONLY_V2; return; } - shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; + shared->storage_run_mode = isPageStorageV2NotExisted(path_pool) ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::MIX_MODE; } PageStorageRunMode Context::getPageStorageRunMode() const @@ -1601,7 +1604,7 @@ bool Context::initializeGlobalStoragePoolIfNeed(const PathPool & path_pool) // Can't init GlobalStoragePool twice. // Because we won't remove the gc task in BackGroundPool // Also won't remove it from ~GlobalStoragePool() - throw Exception("Can't initializeGlobalStoragePool twice", ErrorCodes::LOGICAL_ERROR); + throw Exception("GlobalStoragePool has already been initialized.", ErrorCodes::LOGICAL_ERROR); } if (shared->storage_run_mode == PageStorageRunMode::MIX_MODE || shared->storage_run_mode == PageStorageRunMode::ONLY_V3) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 8ed371c20c6..00ffdebffd0 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -148,7 +148,8 @@ bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_per } StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPool & storage_path_pool, const String & name) - : run_mode(global_ctx.getPageStorageRunMode()) + : logger(Logger::get("StoragePool", name.size() != 0 ? name : DB::toString(ns_id_))) + , run_mode(global_ctx.getPageStorageRunMode()) , ns_id(ns_id_) , global_context(global_ctx) { @@ -264,6 +265,7 @@ PageStorageRunMode StoragePool::restore() // Check number of valid pages in v2 if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) { + LOG_FMT_INFO(logger, "Current pagestorage change from {} to {}", static_cast(PageStorageRunMode::MIX_MODE), static_cast(PageStorageRunMode::ONLY_V3)); run_mode = PageStorageRunMode::ONLY_V3; log_storage_v2 = nullptr; data_storage_v2 = nullptr; @@ -394,8 +396,9 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons break; } // else there is a DTFile with that id, continue to acquire a new ID. - LOG_FMT_WARNING(&Poco::Logger::get(who), - "The DTFile is already exists, continute to acquire another ID. [path={}] [id={}]", + LOG_FMT_WARNING(logger, + "The DTFile is already exists, continute to acquire another ID. [call={}][path={}] [id={}]", + who, existed_path, dtfile_id); } while (true); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 7895da81efb..f030cde0e4c 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -158,6 +159,8 @@ class StoragePool : private boost::noncopyable #ifndef DBMS_PUBLIC_GTEST private: #endif + LoggerPtr logger; + PageStorageRunMode run_mode; // whether the three storage instance is owned by this StoragePool diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index ebcbbc8179a..8ec391c03d4 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -58,7 +58,6 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ testdata_path.swap(absolute_testdata_path); } global_context->initializePathCapacityMetric(0, testdata_path, {}, {}, {}); - global_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2); auto paths = getPathPool(testdata_path); global_context->setPathPool( From ac8428c007cef6f01962527341dabdf9d0885945 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 5 May 2022 18:17:18 +0800 Subject: [PATCH 17/38] fix --- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 4 +-- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 2 +- dbms/src/Storages/Page/V3/PageEntry.h | 11 ++++++++ dbms/src/Storages/Page/V3/PageStorageImpl.cpp | 27 ++++++++++++++++--- .../V3/tests/gtest_page_storage_mix_mode.cpp | 6 +++++ 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index f0cd37e39c2..ca77a352acf 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -210,11 +210,11 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, , hash_salt(++DELTA_MERGE_STORE_HASH_SALT) , log(Logger::get("DeltaMergeStore", fmt::format("{}.{}", db_name, table_name))) { - LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}]", db_name, table_name); - // for mock test, table_id_ should be DB::InvalidTableID NamespaceId ns_id = physical_table_id == DB::InvalidTableID ? TEST_NAMESPACE_ID : physical_table_id; + LOG_FMT_INFO(log, "Restore DeltaMerge Store start [{}.{}] [table_id = {}]", db_name, table_name, physical_table_id); + storage_pool = std::make_shared(global_context, ns_id, path_pool, diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 00ffdebffd0..52643aefe38 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -148,7 +148,7 @@ bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_per } StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPool & storage_path_pool, const String & name) - : logger(Logger::get("StoragePool", name.size() != 0 ? name : DB::toString(ns_id_))) + : logger(Logger::get("StoragePool", !name.empty() ? name : DB::toString(ns_id_))) , run_mode(global_ctx.getPageStorageRunMode()) , ns_id(ns_id_) , global_context(global_ctx) diff --git a/dbms/src/Storages/Page/V3/PageEntry.h b/dbms/src/Storages/Page/V3/PageEntry.h index bbc3915647c..13fa1cd1adb 100644 --- a/dbms/src/Storages/Page/V3/PageEntry.h +++ b/dbms/src/Storages/Page/V3/PageEntry.h @@ -40,6 +40,17 @@ struct PageEntryV3 PageFieldOffsetChecksums field_offsets{}; public: + String toString() const + { + return fmt::format("[file_id={}],[offset={}][size={}],[tag={}],[crc=0x{:X}],[field_offsets size={}]", // + file_id, + offset, + size, + tag, + checksum, + field_offsets.size()); + } + inline bool isValid() const { return file_id != INVALID_BLOBFILE_ID; } size_t getFieldSize(size_t index) const diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp index bb48f18c658..b8ccccbf789 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp @@ -158,7 +158,10 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c PageIdV3Internals page_id_v3s; for (auto p_id : page_ids) + { page_id_v3s.emplace_back(buildV3Id(ns_id, p_id)); + } + if (throw_on_not_exist) { @@ -168,7 +171,12 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c else { auto [page_entries, page_ids_not_found] = page_directory->getOrNull(page_id_v3s, snapshot); - auto page_map = blob_store.read(page_entries, read_limiter); + PageMap page_map = {}; + if (!page_entries.empty()) + { + page_map = blob_store.read(page_entries, read_limiter); + } + for (const auto & page_id_not_found : page_ids_not_found) { Page page_not_found; @@ -193,13 +201,19 @@ PageIds PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c if (throw_on_not_exist) { auto page_entries = page_directory->get(page_id_v3s, snapshot); - blob_store.read(page_entries, handler, read_limiter); + if (!page_entries.empty()) + { + blob_store.read(page_entries, handler, read_limiter); + } return {}; } else { auto [page_entries, page_ids_not_found] = page_directory->getOrNull(page_id_v3s, snapshot); - blob_store.read(page_entries, handler, read_limiter); + if (!page_entries.empty()) + { + blob_store.read(page_entries, handler, read_limiter); + } return page_ids_not_found; } } @@ -216,6 +230,7 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const std::vectorget(buildV3Id(ns_id, page_id), snapshot) : page_directory->getOrNull(buildV3Id(ns_id, page_id), snapshot); + if (entry.isValid()) { auto info = BlobStore::FieldReadInfo(buildV3Id(ns_id, page_id), entry, field_indices); @@ -226,8 +241,12 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const std::vector read_fields; + read_fields.emplace_back(std::make_pair(2, {1, 3, 6})); + ASSERT_NO_THROW(page_reader_mix->read(read_fields)); + } + { // Revert v3 WriteBatch batch; From 3032313e54d164f87109adcc2cf314e81ebcd93e Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 5 May 2022 22:18:39 +0800 Subject: [PATCH 18/38] update --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 44 ++++++++++++++++++- .../V3/tests/gtest_page_storage_mix_mode.cpp | 17 +++++-- dbms/src/Storages/PathPool.cpp | 10 +++-- dbms/src/Storages/PathPool.h | 5 ++- 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 52643aefe38..7b4c56010d5 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -281,6 +281,8 @@ PageStorageRunMode StoragePool::restore() } break; } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } return run_mode; } @@ -300,14 +302,52 @@ void StoragePool::enableGC() void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks & callbacks) { - if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + data_storage_v2->registerExternalPagesCallbacks(callbacks); + break; + } + case PageStorageRunMode::ONLY_V3: + { + data_storage_v3->registerExternalPagesCallbacks(callbacks); + break; + } + case PageStorageRunMode::MIX_MODE: + { data_storage_v2->registerExternalPagesCallbacks(callbacks); + data_storage_v3->registerExternalPagesCallbacks(callbacks); + break; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } void StoragePool::dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) { - if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + data_storage_v2->unregisterExternalPagesCallbacks(ns_id); + break; + } + case PageStorageRunMode::ONLY_V3: + { + data_storage_v3->unregisterExternalPagesCallbacks(ns_id); + break; + } + case PageStorageRunMode::MIX_MODE: + { data_storage_v2->unregisterExternalPagesCallbacks(ns_id); + data_storage_v3->unregisterExternalPagesCallbacks(ns_id); + break; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } } diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 888539bdc04..d60ca432f02 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -245,16 +245,20 @@ try { PageIds page_ids = {1, 2, 3, 4}; - page_ids.emplace_back(1); - page_ids.emplace_back(2); - page_ids.emplace_back(3); - page_ids.emplace_back(4); auto page_maps = page_reader_mix->read(page_ids); ASSERT_EQ(page_maps.size(), 4); ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); ASSERT_PAGE_EQ(c_buff2, buf_sz2, page_maps[3], 3); ASSERT_PAGE_EQ(c_buff2, buf_sz2, page_maps[4], 4); + + // Read page ids which only exited in V2 + page_ids = {1, 2, 7}; + page_maps = page_reader_mix->read(page_ids); + ASSERT_EQ(page_maps.size(), 3); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[7], 7); } { @@ -262,6 +266,10 @@ try PageHandler hander = [](DB::PageId /*page_id*/, const Page & /*page*/) { }; ASSERT_NO_THROW(page_reader_mix->read(page_ids, hander)); + + // Read page ids which only exited in V2 + page_ids = {1, 2, 7}; + ASSERT_NO_THROW(page_reader_mix->read(page_ids, hander)); } { @@ -280,6 +288,7 @@ try } { + // Read page ids which only exited in V2 std::vector read_fields; read_fields.emplace_back(std::make_pair(2, {1, 3, 6})); ASSERT_NO_THROW(page_reader_mix->read(read_fields)); diff --git a/dbms/src/Storages/PathPool.cpp b/dbms/src/Storages/PathPool.cpp index 5a542ba0eea..0ae42ec7313 100644 --- a/dbms/src/Storages/PathPool.cpp +++ b/dbms/src/Storages/PathPool.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -24,7 +25,6 @@ #include #include #include -#include #include #include @@ -64,7 +64,7 @@ PathPool::PathPool( , enable_raft_compatible_mode(enable_raft_compatible_mode_) , global_capacity(global_capacity_) , file_provider(file_provider_) - , log(&Poco::Logger::get("PathPool")) + , log(Logger::get("PathPool")) { if (kvstore_paths.empty()) { @@ -134,7 +134,7 @@ StoragePathPool::StoragePathPool( // , path_need_database_name(path_need_database_name_) , global_capacity(std::move(global_capacity_)) , file_provider(std::move(file_provider_)) - , log(&Poco::Logger::get("StoragePathPool")) + , log(Logger::get("StoragePathPool")) { if (unlikely(database.empty() || table.empty())) throw Exception(fmt::format("Can NOT create StoragePathPool [database={}] [table={}]", database, table), ErrorCodes::LOGICAL_ERROR); @@ -240,6 +240,7 @@ void StoragePathPool::drop(bool recursive, bool must_success) { if (Poco::File dir(path_info.path); dir.exists()) { + LOG_FMT_INFO(log, "Begin to drop [dir={}] from main_path_infos", path_info.path); file_provider->deleteDirectory(dir.path(), false, recursive); // update global used size @@ -269,6 +270,7 @@ void StoragePathPool::drop(bool recursive, bool must_success) { if (Poco::File dir(path_info.path); dir.exists()) { + LOG_FMT_INFO(log, "Begin to drop [dir={}] from latest_path_infos", path_info.path); file_provider->deleteDirectory(dir.path(), false, recursive); // When PageStorage is dropped, it will update the size in global_capacity. @@ -318,7 +320,7 @@ String genericChoosePath(const std::vector & paths, // const PathCapacityMetricsPtr & global_capacity, // std::function & paths, size_t idx)> path_generator, // std::function path_getter, // - Poco::Logger * log, // + LoggerPtr log, // const String & log_msg) { if (paths.size() == 1) diff --git a/dbms/src/Storages/PathPool.h b/dbms/src/Storages/PathPool.h index fd7b70a4bbd..40a3cb9a636 100644 --- a/dbms/src/Storages/PathPool.h +++ b/dbms/src/Storages/PathPool.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include @@ -109,7 +110,7 @@ class PathPool FileProviderPtr file_provider; - Poco::Logger * log; + LoggerPtr log; }; class StableDiskDelegator : private boost::noncopyable @@ -453,7 +454,7 @@ class StoragePathPool FileProviderPtr file_provider; - Poco::Logger * log; + LoggerPtr log; }; } // namespace DB From 9e1d0e7150347f1256926af8a4b053992b21ceee Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 5 May 2022 23:21:43 +0800 Subject: [PATCH 19/38] fix external page not right --- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 2 +- .../src/Storages/DeltaMerge/DeltaMergeStore.h | 4 ++-- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 22 ++++++++++++++++--- .../tests/gtest_dm_delta_merge_store.cpp | 4 ++-- .../DeltaMerge/tests/gtest_dm_segment.cpp | 2 +- dbms/src/Storages/Page/PageStorage.h | 2 ++ dbms/src/Storages/Page/V2/PageStorage.cpp | 13 +++++++++++ dbms/src/Storages/Page/V2/PageStorage.h | 2 ++ dbms/src/Storages/Page/V3/PageStorageImpl.cpp | 7 +++++- dbms/src/Storages/Page/V3/PageStorageImpl.h | 2 ++ 10 files changed, 50 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index ca77a352acf..a02ba37b1ac 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -667,7 +667,7 @@ void DeltaMergeStore::preIngestFile(const String & parent_path, const PageId fil void DeltaMergeStore::ingestFiles( const DMContextPtr & dm_context, const RowKeyRange & range, - const std::vector & file_ids, + const PageIds & file_ids, bool clear_data_in_range) { if (unlikely(shutdown_called.load(std::memory_order_relaxed))) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 4f831ddfe0e..7ae06f0cd55 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -328,13 +328,13 @@ class DeltaMergeStore : private boost::noncopyable void ingestFiles(const DMContextPtr & dm_context, // const RowKeyRange & range, - const std::vector & file_ids, + const PageIds & file_ids, bool clear_data_in_range); void ingestFiles(const Context & db_context, // const DB::Settings & db_settings, const RowKeyRange & range, - const std::vector & file_ids, + const PageIds & file_ids, bool clear_data_in_range) { auto dm_context = newDMContext(db_context, db_settings); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 7b4c56010d5..018e6358f5c 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -316,8 +316,24 @@ void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks } case PageStorageRunMode::MIX_MODE: { - data_storage_v2->registerExternalPagesCallbacks(callbacks); - data_storage_v3->registerExternalPagesCallbacks(callbacks); + // When PageStorage run as Mix Mode. + // We need both get alive pages from V2 and V3 which will feedback for the DM. + // But V2 and V3 won't GC in the same time. So V3 need proxy V2 external pages callback. + // When V3 GC happend, scan the external pages from V3, in remover will scanner all of external pages from V2. + ExternalPageCallbacks mix_mode_callbacks; + + mix_mode_callbacks.scanner = [callbacks]() { + return callbacks.scanner(); + }; + + mix_mode_callbacks.remover = [this, callbacks](const ExternalPageCallbacks::PathAndIdsVec & path_and_ids_vec, const std::set & valid_ids) { + // ns_id won't used on V2 + auto v2_valid_page_ids = data_storage_v2->getAliveExternalPageIds(ns_id); + v2_valid_page_ids.insert(valid_ids.begin(), valid_ids.end()); + callbacks.remover(path_and_ids_vec, v2_valid_page_ids); + }; + mix_mode_callbacks.ns_id = ns_id; + data_storage_v3->registerExternalPagesCallbacks(mix_mode_callbacks); break; } default: @@ -341,7 +357,7 @@ void StoragePool::dataUnregisterExternalPagesCallbacks(NamespaceId ns_id) } case PageStorageRunMode::MIX_MODE: { - data_storage_v2->unregisterExternalPagesCallbacks(ns_id); + // no need unregister callback in V2. data_storage_v3->unregisterExternalPagesCallbacks(ns_id); break; } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index bb0e47bddbd..b759d1fe9ef 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -178,7 +178,7 @@ class DeltaMergeStoreRWTest return s; } - std::pair> genDMFile(DMContext & context, const Block & block) + std::pair genDMFile(DMContext & context, const Block & block) { auto input_stream = std::make_shared(block); auto [store_path, file_id] = store->preAllocateIngestFile(); @@ -1344,7 +1344,7 @@ try { auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); - std::vector file_ids; + PageIds file_ids; auto ingest_range = RowKeyRange::fromHandleRange(HandleRange{32, 256}); store->ingestFiles(dm_context, ingest_range, file_ids, /*clear_data_in_range*/ true); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 6f1da767923..6bf33465366 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -1280,7 +1280,7 @@ class Segment_test_2 : public Segment_test Segment_test::SetUp(); } - std::pair> genDMFile(DMContext & context, const Block & block) + std::pair genDMFile(DMContext & context, const Block & block) { auto delegator = context.path_pool.getStableDiskDelegator(); auto file_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index cd9e4eb7604..0a20b110244 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -237,6 +237,8 @@ class PageStorage : private boost::noncopyable virtual size_t getNumberOfPages() = 0; + virtual std::set getAliveExternalPageIds(NamespaceId ns_id) = 0; + void write(WriteBatch && write_batch, const WriteLimiterPtr & write_limiter = nullptr) { writeImpl(std::move(write_batch), write_limiter); diff --git a/dbms/src/Storages/Page/V2/PageStorage.cpp b/dbms/src/Storages/Page/V2/PageStorage.cpp index 32367c4e708..8af76f1739a 100644 --- a/dbms/src/Storages/Page/V2/PageStorage.cpp +++ b/dbms/src/Storages/Page/V2/PageStorage.cpp @@ -611,6 +611,19 @@ size_t PageStorage::getNumberOfPages() } } +std::set PageStorage::getAliveExternalPageIds(NamespaceId /*ns_id*/) +{ + const auto & concrete_snap = getConcreteSnapshot(); + if (concrete_snap) + { + return concrete_snap->version()->validNormalPageIds(); + } + else + { + throw Exception("Can't get concrete snapshot", ErrorCodes::LOGICAL_ERROR); + } +} + DB::Page PageStorage::readImpl(NamespaceId /*ns_id*/, PageId page_id, const ReadLimiterPtr & read_limiter, SnapshotPtr snapshot, bool throw_on_not_exist) { if (!snapshot) diff --git a/dbms/src/Storages/Page/V2/PageStorage.h b/dbms/src/Storages/Page/V2/PageStorage.h index 6cefb6407d2..a80fe8f2a3f 100644 --- a/dbms/src/Storages/Page/V2/PageStorage.h +++ b/dbms/src/Storages/Page/V2/PageStorage.h @@ -109,6 +109,8 @@ class PageStorage : public DB::PageStorage size_t getNumberOfPages() override; + std::set getAliveExternalPageIds(NamespaceId ns_id) override; + void writeImpl(DB::WriteBatch && wb, const WriteLimiterPtr & write_limiter) override; DB::PageEntry getEntryImpl(NamespaceId ns_id, PageId page_id, SnapshotPtr snapshot) override; diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp index b8ccccbf789..14cc666e5f6 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp @@ -91,6 +91,11 @@ size_t PageStorageImpl::getNumberOfPages() return page_directory->numPages(); } +std::set PageStorageImpl::getAliveExternalPageIds(NamespaceId ns_id) +{ + return page_directory->getAliveExternalIds(ns_id); +} + void PageStorageImpl::writeImpl(DB::WriteBatch && write_batch, const WriteLimiterPtr & write_limiter) { if (unlikely(write_batch.empty())) @@ -299,7 +304,7 @@ bool PageStorageImpl::gcImpl(bool /*not_skip*/, const WriteLimiterPtr & write_li for (const auto & [ns_id, callbacks] : callbacks_container) { auto pending_external_pages = callbacks.scanner(); - auto alive_external_ids = page_directory->getAliveExternalIds(ns_id); + auto alive_external_ids = getAliveExternalPageIds(ns_id); callbacks.remover(pending_external_pages, alive_external_ids); } } diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.h b/dbms/src/Storages/Page/V3/PageStorageImpl.h index 272cbf73a7d..b687aaf8cd6 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.h +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.h @@ -73,6 +73,8 @@ class PageStorageImpl : public DB::PageStorage size_t getNumberOfPages() override; + std::set getAliveExternalPageIds(NamespaceId ns_id) override; + void writeImpl(DB::WriteBatch && write_batch, const WriteLimiterPtr & write_limiter) override; DB::PageEntry getEntryImpl(NamespaceId ns_id, PageId page_id, SnapshotPtr snapshot) override; From 570ef9a65cd8b8d751018d919dcf6dd90b7d9730 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 6 May 2022 23:23:27 +0800 Subject: [PATCH 20/38] update --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 2 -- dbms/src/Storages/DeltaMerge/StoragePool.h | 1 - dbms/src/Storages/Page/PageStorage.h | 8 +++----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 018e6358f5c..3843ddb709a 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -70,8 +70,6 @@ PageStorage::Config extractConfig(const Settings & settings, StorageType subtype return config; } - -std::shared_ptr GlobalStoragePool::global_storage_pool = nullptr; GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings) : // The iops and bandwidth in log_storage are relatively high, use multi-disks if possible log_storage(PageStorage::create("__global__.log", diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index f030cde0e4c..232cfd9207e 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -56,7 +56,6 @@ class GlobalStoragePool : private boost::noncopyable bool gc(const Settings & settings, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); private: - static std::shared_ptr global_storage_pool; PageStoragePtr log_storage; PageStoragePtr data_storage; PageStoragePtr meta_storage; diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 0a20b110244..b349b3b5d99 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -381,7 +381,7 @@ class PageReader : private boost::noncopyable case PageStorageRunMode::MIX_MODE: { const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); - if (page_from_v3.page_id != INVALID_PAGE_ID) + if (page_from_v3.isValid()) { return page_from_v3; } @@ -410,8 +410,7 @@ class PageReader : private boost::noncopyable PageIds invalid_page_ids; for (const auto & [query_page_id, page] : page_maps) { - // TODO : change to isvalid(), after pre pr merged. - if (page.page_id == INVALID_PAGE_ID) + if (!page.isValid()) { invalid_page_ids.emplace_back(query_page_id); } @@ -479,8 +478,7 @@ class PageReader : private boost::noncopyable for (const auto & page_field : page_fields) { - // TODO : change to isvalid(), after pre pr merged. - if (page_maps[page_field.first].page_id == INVALID_PAGE_ID) + if (!page_maps[page_field.first].isValid()) { invalid_page_fields.emplace_back(page_field); } From 3486fcba195b8245758b45907f79fc9a12d837e3 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sat, 7 May 2022 11:02:07 +0800 Subject: [PATCH 21/38] update version check --- dbms/src/Interpreters/Context.cpp | 51 +++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 7434eca0e61..69aaa329932 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1548,9 +1548,8 @@ ReadLimiterPtr Context::getReadLimiter() const return getIORateLimiter().getReadLimiter(); } -static bool isPageStorageV2NotExisted(const PathPool & path_pool) +static bool isKVStoreEmpty(const PathPool & path_pool) { - // Check whether there are any files in kvstore path, if exists, then this is not a new node. for (const auto & path : path_pool.listKVStorePaths()) { Poco::File dir(path); @@ -1567,6 +1566,52 @@ static bool isPageStorageV2NotExisted(const PathPool & path_pool) return true; } +static bool isPageStorageV2Existed(const PathPool & path_pool) +{ + for (const auto & path : path_pool.listKVStorePaths()) + { + Poco::File dir(path); + if (!dir.exists()) + continue; + + std::vector files; + dir.list(files); + if (!files.empty()) + { + for (const auto file_name : files) + { + const auto & find_index = file_name.find("page"); + if (find_index != std::string::npos) + { + return true; + } + } + // KVStore is not empty, but can't find any of v2 data in it. + } + } + + // If not data in KVStore. It means V2 data must not existed. + return false; +} + +static bool isPageStorageV3Existed(const PathPool & path_pool) +{ + for (const auto & path : path_pool.listGlobalPagePaths()) + { + Poco::File dir(path); + if (!dir.exists()) + continue; + + std::vector files; + dir.list(files); + if (!files.empty()) + { + return true; + } + } + return false; +} + void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3) { auto lock = getLock(); @@ -1581,7 +1626,7 @@ void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ return; } - shared->storage_run_mode = isPageStorageV2NotExisted(path_pool) ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::MIX_MODE; + shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; } PageStorageRunMode Context::getPageStorageRunMode() const From ca4d0784f31e5a748f8e62b98655ba8c407a48a7 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sat, 7 May 2022 13:41:28 +0800 Subject: [PATCH 22/38] removed enable_ps_v3 in main and port this config into format_version --- dbms/src/Interpreters/Context.cpp | 53 +++++++++++++------------ dbms/src/Interpreters/Context.h | 2 +- dbms/src/Server/Server.cpp | 2 +- dbms/src/Server/StorageConfigParser.cpp | 5 +-- dbms/src/Server/StorageConfigParser.h | 1 - dbms/src/Storages/FormatVersion.h | 19 ++++++++- 6 files changed, 48 insertions(+), 34 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 69aaa329932..b083a0d2dee 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1548,23 +1548,6 @@ ReadLimiterPtr Context::getReadLimiter() const return getIORateLimiter().getReadLimiter(); } -static bool isKVStoreEmpty(const PathPool & path_pool) -{ - for (const auto & path : path_pool.listKVStorePaths()) - { - Poco::File dir(path); - if (!dir.exists()) - continue; - - std::vector files; - dir.list(files); - if (!files.empty()) - { - return false; - } - } - return true; -} static bool isPageStorageV2Existed(const PathPool & path_pool) { @@ -1578,7 +1561,7 @@ static bool isPageStorageV2Existed(const PathPool & path_pool) dir.list(files); if (!files.empty()) { - for (const auto file_name : files) + for (const auto & file_name : files) { const auto & find_index = file_name.find("page"); if (find_index != std::string::npos) @@ -1612,21 +1595,39 @@ static bool isPageStorageV3Existed(const PathPool & path_pool) return false; } -void Context::initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3) +void Context::initializePageStorageMode(const PathPool & path_pool, UInt64 storage_page_format_version) { auto lock = getLock(); - // If it's a new node also enable_ps_v3 is 1, then we direct use ONLY_V3. - // If it's not a new node also enable_ps_v3 is 1, then we use MIX_MODE. - // If it's not a new node also enable_ps_v3 is 0, then we use Only_V2. - // If it's a new node also enable_ps_v3 is 0, then we use Only_V2. - if (!enable_ps_v3) + /** + * PageFormat::V2 + isPageStorageV3Existed is false + whatever isPageStorageV2Existed true or false = ONLY_V2 + * PageFormat::V2 + isPageStorageV3Existed is true + whatever isPageStorageV2Existed true or false = ERROR Config + * PageFormat::V3 + isPageStorageV2Existed is true + whatever isPageStorageV3Existed true or false = MIX_MODE + * PageFormat::V3 + isPageStorageV2Existed is false + whatever isPageStorageV3Existed true or false = ONLY_V3 + */ + + switch (storage_page_format_version) { + case PageFormat::V2: + { + if (isPageStorageV3Existed(path_pool)) + { + throw Exception("Invalid config `storage.format_version`, Current page V3 data exist. But using the PageFormat::V2", + ErrorCodes::LOGICAL_ERROR); + } + // not exist V3 shared->storage_run_mode = PageStorageRunMode::ONLY_V2; return; } - - shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; + case PageFormat::V3: + { + shared->storage_run_mode = isPageStorageV2Existed(path_pool) ? PageStorageRunMode::MIX_MODE : PageStorageRunMode::ONLY_V3; + return; + } + default: + throw Exception(fmt::format("Can't detect the format version of Page [page_version={}]", storage_page_format_version), + ErrorCodes::LOGICAL_ERROR); + } } PageStorageRunMode Context::getPageStorageRunMode() const diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 922a9047013..63aefcbece9 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -406,7 +406,7 @@ class Context ReadLimiterPtr getReadLimiter() const; IORateLimiter & getIORateLimiter() const; - void initializePageStorageMode(const PathPool & path_pool, bool enable_ps_v3); + void initializePageStorageMode(const PathPool & path_pool, UInt64 storage_page_format_version); void setPageStorageRunMode(PageStorageRunMode run_mode) const; PageStorageRunMode getPageStorageRunMode() const; bool initializeGlobalStoragePoolIfNeed(const PathPool & path_pool); diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index ecd69805e97..705b8a533f3 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1113,7 +1113,7 @@ int Server::main(const std::vector & /*args*/) global_context->getPathCapacity(), global_context->getFileProvider()); - global_context->initializePageStorageMode(global_context->getPathPool(), storage_config.enable_ps_v3); + global_context->initializePageStorageMode(global_context->getPathPool(), STORAGE_FORMAT_CURRENT.page); global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); LOG_FMT_INFO(log, "Global PageStorage run mode is {}", static_cast(global_context->getPageStorageRunMode())); diff --git a/dbms/src/Server/StorageConfigParser.cpp b/dbms/src/Server/StorageConfigParser.cpp index 653a4eb947f..d43ccb850f1 100644 --- a/dbms/src/Server/StorageConfigParser.cpp +++ b/dbms/src/Server/StorageConfigParser.cpp @@ -213,11 +213,8 @@ void TiFlashStorageConfig::parseMisc(const String & storage_section, Poco::Logge }; lazily_init_store = get_bool_config_or_default("lazily_init_store", lazily_init_store); - // config for experimental feature, may remove later - enable_ps_v3 = get_bool_config_or_default("enable_ps_v3", enable_ps_v3); - - LOG_FMT_INFO(log, "format_version {} lazily_init_store {} enable_ps_v3 {}", format_version, lazily_init_store, enable_ps_v3); + LOG_FMT_INFO(log, "format_version {} lazily_init_store {}", format_version, lazily_init_store); } Strings TiFlashStorageConfig::getAllNormalPaths() const diff --git a/dbms/src/Server/StorageConfigParser.h b/dbms/src/Server/StorageConfigParser.h index 65df7be3174..4efc5637634 100644 --- a/dbms/src/Server/StorageConfigParser.h +++ b/dbms/src/Server/StorageConfigParser.h @@ -103,7 +103,6 @@ struct TiFlashStorageConfig UInt64 format_version = 0; bool lazily_init_store = true; - bool enable_ps_v3 = true; public: TiFlashStorageConfig() = default; diff --git a/dbms/src/Storages/FormatVersion.h b/dbms/src/Storages/FormatVersion.h index 4095ffb1498..dbe3d9ee9bc 100644 --- a/dbms/src/Storages/FormatVersion.h +++ b/dbms/src/Storages/FormatVersion.h @@ -64,6 +64,11 @@ using Version = UInt32; inline static constexpr Version V1 = 1; // Support multiple thread-write && read with offset inside page. See FLASH_341 && FLASH-942 for details. inline static constexpr Version V2 = 2; +// Support multiple thread-write/read with offset inside page && support wal store meta && support space reused. +// If we do enabled PageFormat::V3, it is not means all data in Disk will be V3 format. +// - If we already have V2 data in disk. It will turn PageStorage into MIX_MODE +// - If we don't have any v2 data in disk. It will turn PageStorage into ONLY_V3 +inline static constexpr Version V3 = 3; } // namespace PageFormat struct StorageFormatVersion @@ -103,7 +108,17 @@ inline static const StorageFormatVersion STORAGE_FORMAT_V3 = StorageFormatVersio .identifier = 3, }; -inline StorageFormatVersion STORAGE_FORMAT_CURRENT = STORAGE_FORMAT_V3; + +inline static const StorageFormatVersion STORAGE_FORMAT_V4 = StorageFormatVersion{ + .segment = SegmentFormat::V2, + .dm_file = DMFileFormat::V2, + .stable = StableFormat::V1, + .delta = DeltaFormat::V3, + .page = PageFormat::V3, // diff + .identifier = 4, +}; + +inline StorageFormatVersion STORAGE_FORMAT_CURRENT = STORAGE_FORMAT_V4; inline const StorageFormatVersion & toStorageFormat(UInt64 setting) { @@ -115,6 +130,8 @@ inline const StorageFormatVersion & toStorageFormat(UInt64 setting) return STORAGE_FORMAT_V2; case 3: return STORAGE_FORMAT_V3; + case 4: + return STORAGE_FORMAT_V4; default: throw Exception("Illegal setting value: " + DB::toString(setting)); } From efe69d888fba81a6494fea99cd1c9d3ec152d658 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sat, 7 May 2022 18:09:07 +0800 Subject: [PATCH 23/38] fix version not right --- dbms/src/Storages/Page/V2/PageFile.cpp | 2 +- dbms/src/Storages/Page/V2/PageStorage.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/Page/V2/PageFile.cpp b/dbms/src/Storages/Page/V2/PageFile.cpp index 60846685eb5..5f2ef7c26e0 100644 --- a/dbms/src/Storages/Page/V2/PageFile.cpp +++ b/dbms/src/Storages/Page/V2/PageFile.cpp @@ -134,7 +134,7 @@ std::pair genWriteData( // char * data_pos = data_buffer; PageUtil::put(meta_pos, meta_write_bytes); - PageUtil::put(meta_pos, STORAGE_FORMAT_CURRENT.page); + PageUtil::put(meta_pos, PageFormat::V2); PageUtil::put(meta_pos, wb.getSequence()); PageOffset page_data_file_off = page_file.getDataFileAppendPos(); diff --git a/dbms/src/Storages/Page/V2/PageStorage.h b/dbms/src/Storages/Page/V2/PageStorage.h index a80fe8f2a3f..cb55a769f37 100644 --- a/dbms/src/Storages/Page/V2/PageStorage.h +++ b/dbms/src/Storages/Page/V2/PageStorage.h @@ -148,11 +148,11 @@ class PageStorage : public DB::PageStorage option.remove_tmp_files = false; auto page_files = listAllPageFiles(file_provider, delegator, log, option); if (page_files.empty()) - return STORAGE_FORMAT_CURRENT.page; + return PageFormat::V2; bool all_empty = true; PageFormat::Version max_binary_version = PageFormat::V1; - PageFormat::Version temp_version = STORAGE_FORMAT_CURRENT.page; + PageFormat::Version temp_version = PageFormat::V2; for (auto iter = page_files.rbegin(); iter != page_files.rend(); ++iter) { // Skip those files without valid meta @@ -171,7 +171,7 @@ class PageStorage : public DB::PageStorage LOG_FMT_DEBUG(log, "getMaxDataVersion done from {} [max version={}]", reader->toString(), max_binary_version); break; } - max_binary_version = (all_empty ? STORAGE_FORMAT_CURRENT.page : max_binary_version); + max_binary_version = (all_empty ? PageFormat::V2 : max_binary_version); return max_binary_version; } From 4a45f87d5bbd7edc31d459236ce41939325e0e05 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Sun, 8 May 2022 11:49:28 +0800 Subject: [PATCH 24/38] fix dump trav --- dbms/src/Storages/Page/PageStorage.h | 10 ++++++---- dbms/src/Storages/Transaction/RegionPersister.cpp | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 4399e9cefe6..62743abb0f2 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -629,18 +629,20 @@ class PageReader : private boost::noncopyable { case PageStorageRunMode::ONLY_V2: { - storage_v2->traverse(acceptor, snap); + storage_v2->traverse(acceptor, nullptr); break; } case PageStorageRunMode::ONLY_V3: { - storage_v3->traverse(acceptor, snap); + storage_v3->traverse(acceptor, nullptr); break; } case PageStorageRunMode::MIX_MODE: { - storage_v2->traverse(acceptor, toConcreteV3Snapshot()); - storage_v3->traverse(acceptor, toConcreteV2Snapshot()); + // Used by RegionPersister::restore + // Must traverse storage_v3 before storage_v2 + storage_v3->traverse(acceptor, toConcreteV3Snapshot()); + storage_v2->traverse(acceptor, toConcreteV2Snapshot()); break; } default: diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index 66e46699d34..dddcedc4c1a 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -269,10 +269,19 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, if (page_reader) { auto acceptor = [&](const DB::Page & page) { + // We will traverse the pages in V3 before traverse the pages in V2 When we used MIX MODE + // So if we found the page_id have been traversed, just skip it. + if (const auto it = regions.find(page.page_id); it != regions.end()) + { + LOG_FMT_INFO(log, "Already exist [page_id={}], skip it.", page.page_id); + return; + } + ReadBufferFromMemory buf(page.data.begin(), page.data.size()); auto region = Region::deserialize(buf, proxy_helper); if (page.page_id != region->id()) throw Exception("region id and page id not match!", ErrorCodes::LOGICAL_ERROR); + regions.emplace(page.page_id, region); }; page_reader->traverse(acceptor); From e80b115adc0e8759b327a695de46e11fbab195c2 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Mon, 9 May 2022 14:15:15 +0800 Subject: [PATCH 25/38] fix v2 and v3 used same path in kvstore --- dbms/src/Storages/Transaction/RegionPersister.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index dddcedc4c1a..b1f3ec77e4a 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -222,7 +222,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, auto page_storage_v3 = std::make_shared( // "RegionPersister", - delegator, + path_pool.getPSDiskDelegatorGlobalMulti("kvstore"), config, provider); page_storage_v3->restore(); @@ -237,9 +237,11 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, delegator, config, provider); + // V3 should not used getPSDiskDelegatorRaft + // Because V2 will delete all invalid(unrecognized) file when it restore auto page_storage_v3 = std::make_shared( // "RegionPersister", - delegator, + path_pool.getPSDiskDelegatorGlobalMulti("kvstore"), config, provider); From c5c222d2331e9efe5ca117c914666e94d4e188bb Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Mon, 9 May 2022 23:01:21 +0800 Subject: [PATCH 26/38] update --- dbms/src/Interpreters/Context.cpp | 3 ++- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index b083a0d2dee..9733f665e22 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1612,7 +1612,8 @@ void Context::initializePageStorageMode(const PathPool & path_pool, UInt64 stora { if (isPageStorageV3Existed(path_pool)) { - throw Exception("Invalid config `storage.format_version`, Current page V3 data exist. But using the PageFormat::V2", + throw Exception("Invalid config `storage.format_version`, Current page V3 data exist. But using the PageFormat::V2." + "If you are downgrading the format_version for this TiFlash node, you need to rebuild the data from scratch.", ErrorCodes::LOGICAL_ERROR); } // not exist V3 diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index e024c951110..8b02baebc4a 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -257,10 +257,6 @@ PageStorageRunMode StoragePool::restore() } case PageStorageRunMode::MIX_MODE: { - max_log_page_id = std::max(log_storage_v2->getMaxId(ns_id), log_storage_v3->getMaxId(ns_id)); - max_data_page_id = std::max(data_storage_v2->getMaxId(ns_id), data_storage_v3->getMaxId(ns_id)); - max_meta_page_id = std::max(meta_storage_v2->getMaxId(ns_id), meta_storage_v3->getMaxId(ns_id)); - // Check number of valid pages in v2 if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) { @@ -277,6 +273,16 @@ PageStorageRunMode StoragePool::restore() log_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, log_storage_v3); data_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, data_storage_v3); meta_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, meta_storage_v3); + + max_log_page_id = log_storage_v3->getMaxId(ns_id); + max_data_page_id = data_storage_v3->getMaxId(ns_id); + max_meta_page_id = meta_storage_v3->getMaxId(ns_id); + } + else + { + max_log_page_id = std::max(log_storage_v2->getMaxId(ns_id), log_storage_v3->getMaxId(ns_id)); + max_data_page_id = std::max(data_storage_v2->getMaxId(ns_id), data_storage_v3->getMaxId(ns_id)); + max_meta_page_id = std::max(meta_storage_v2->getMaxId(ns_id), meta_storage_v3->getMaxId(ns_id)); } break; } @@ -321,10 +327,7 @@ void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks // When V3 GC happend, scan the external pages from V3, in remover will scanner all of external pages from V2. ExternalPageCallbacks mix_mode_callbacks; - mix_mode_callbacks.scanner = [callbacks]() { - return callbacks.scanner(); - }; - + mix_mode_callbacks.scanner = callbacks.scanner; mix_mode_callbacks.remover = [this, callbacks](const ExternalPageCallbacks::PathAndIdsVec & path_and_ids_vec, const std::set & valid_ids) { // ns_id won't used on V2 auto v2_valid_page_ids = data_storage_v2->getAliveExternalPageIds(ns_id); From 30fa576a6be69cb1bcca45d489a87a5e4b614108 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 10 May 2022 14:06:03 +0800 Subject: [PATCH 27/38] up --- dbms/src/Storages/Transaction/RegionPersister.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionPersister.cpp b/dbms/src/Storages/Transaction/RegionPersister.cpp index b1f3ec77e4a..77901922e2f 100644 --- a/dbms/src/Storages/Transaction/RegionPersister.cpp +++ b/dbms/src/Storages/Transaction/RegionPersister.cpp @@ -176,7 +176,6 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, auto & path_pool = global_context.getPathPool(); auto delegator = path_pool.getPSDiskDelegatorRaft(); auto provider = global_context.getFileProvider(); - // If the GlobalStoragePool is initialized, then use v3 format auto run_mode = global_context.getPageStorageRunMode(); switch (run_mode) @@ -272,7 +271,7 @@ RegionMap RegionPersister::restore(const TiFlashRaftProxyHelper * proxy_helper, { auto acceptor = [&](const DB::Page & page) { // We will traverse the pages in V3 before traverse the pages in V2 When we used MIX MODE - // So if we found the page_id have been traversed, just skip it. + // If we found the page_id has been restored, just skip it. if (const auto it = regions.find(page.page_id); it != regions.end()) { LOG_FMT_INFO(log, "Already exist [page_id={}], skip it.", page.page_id); From 45f7060ab9c409d2d9053156e562e27d65bf9267 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 10 May 2022 18:46:13 +0800 Subject: [PATCH 28/38] update --- dbms/src/Interpreters/Context.cpp | 1 + dbms/src/Storages/Page/V3/PageDirectory.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 9733f665e22..25dffe263fd 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1608,6 +1608,7 @@ void Context::initializePageStorageMode(const PathPool & path_pool, UInt64 stora switch (storage_page_format_version) { + case PageFormat::V1: case PageFormat::V2: { if (isPageStorageV3Existed(path_pool)) diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index e4351ef9e4b..fa4a9f27e39 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -831,7 +831,7 @@ PageIdV3Internal PageDirectory::getNormalPageId(PageIdV3Internal page_id, const } else { - return PageIdV3Internal(INVALID_PAGE_ID, 0); + return buildV3Id(0, INVALID_PAGE_ID); } } } @@ -868,7 +868,7 @@ PageIdV3Internal PageDirectory::getNormalPageId(PageIdV3Internal page_id, const } else { - return PageIdV3Internal(INVALID_PAGE_ID, 0); + return buildV3Id(0, INVALID_PAGE_ID); } } From ddfd8e3bfa150c1166d57fbcd42ef50486422bdb Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 10 May 2022 18:47:43 +0800 Subject: [PATCH 29/38] change data to multi --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 8b02baebc4a..bbc179a026c 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -207,7 +207,7 @@ StoragePool::StoragePool(Context & global_ctx, NamespaceId ns_id_, StoragePathPo extractConfig(global_context.getSettingsRef(), StorageType::Log), global_context.getFileProvider()); data_storage_v2 = PageStorage::create(name + ".data", - storage_path_pool.getPSDiskDelegatorSingle("data"), + storage_path_pool.getPSDiskDelegatorMulti("data"), extractConfig(global_context.getSettingsRef(), StorageType::Data), global_ctx.getFileProvider()); meta_storage_v2 = PageStorage::create(name + ".meta", From fc12039ff49b91fbea522aac5869bc07795acf10 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 11 May 2022 12:58:00 +0800 Subject: [PATCH 30/38] Update dbms/src/Storages/DeltaMerge/StoragePool.cpp Co-authored-by: JaySon --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index bbc179a026c..11174f278f1 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -299,6 +299,7 @@ StoragePool::~StoragePool() void StoragePool::enableGC() { + // The data in V3 will be GCed by `GlobalStoragePool::gc`, only register gc task under only v2/mix mode if (run_mode == PageStorageRunMode::ONLY_V2 || run_mode == PageStorageRunMode::MIX_MODE) { gc_handle = global_context.getBackgroundPool().addTask([this] { return this->gc(global_context.getSettingsRef()); }); From c0a6e58502afe331a24f84e2e2fb5cb102721ed6 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 11 May 2022 16:20:08 +0800 Subject: [PATCH 31/38] fix build --- dbms/src/Debug/dbgFuncMisc.cpp | 4 +- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 50 +- dbms/src/Storages/DeltaMerge/StoragePool.h | 42 +- dbms/src/Storages/Page/PageStorage.cpp | 502 +++++++++++++++++++ dbms/src/Storages/Page/PageStorage.h | 477 +----------------- 5 files changed, 584 insertions(+), 491 deletions(-) diff --git a/dbms/src/Debug/dbgFuncMisc.cpp b/dbms/src/Debug/dbgFuncMisc.cpp index b9f62317189..e43ca5dd725 100644 --- a/dbms/src/Debug/dbgFuncMisc.cpp +++ b/dbms/src/Debug/dbgFuncMisc.cpp @@ -104,9 +104,7 @@ void dbgFuncTriggerGlobalPageStorageGC(Context & context, const ASTs & /*args*/, auto global_storage_pool = context.getGlobalStoragePool(); if (global_storage_pool) { - global_storage_pool->meta()->gc(); - global_storage_pool->log()->gc(); - global_storage_pool->data()->gc(); + global_storage_pool->gc(); } } } // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index c2d30ef3475..90a9d578e15 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -119,12 +119,20 @@ void GlobalStoragePool::restore() false); } -bool GlobalStoragePool::gc(const Settings & settings, const Seconds & try_gc_period) +bool GlobalStoragePool::gc() +{ + return gc(Settings(), true, DELTA_MERGE_GC_PERIOD); +} + +bool GlobalStoragePool::gc(const Settings & settings, bool immediately, const Seconds & try_gc_period) { - // No need lock Timepoint now = Clock::now(); - if (now < (last_try_gc_time.load() + try_gc_period)) - return false; + if (!immediately) + { + // No need lock + if (now < (last_try_gc_time.load() + try_gc_period)) + return false; + } last_try_gc_time = now; @@ -237,9 +245,9 @@ PageStorageRunMode StoragePool::restore() { case PageStorageRunMode::ONLY_V2: { - auto log_max_ids = log_storage->restore(); - auto data_max_ids = data_storage->restore(); - auto meta_max_ids = meta_storage->restore(); + auto log_max_ids = log_storage_v2->restore(); + auto data_max_ids = data_storage_v2->restore(); + auto meta_max_ids = meta_storage_v2->restore(); assert(log_max_ids.size() == 1); assert(data_max_ids.size() == 1); @@ -253,9 +261,9 @@ PageStorageRunMode StoragePool::restore() } case PageStorageRunMode::ONLY_V3: { - max_log_page_id = global_storage_pool.getLogMaxId(ns_id); - max_data_page_id = global_storage_pool.getDataMaxId(ns_id); - max_meta_page_id = global_storage_pool.getMetaMaxId(ns_id); + max_log_page_id = global_storage_pool->getLogMaxId(ns_id); + max_data_page_id = global_storage_pool->getDataMaxId(ns_id); + max_meta_page_id = global_storage_pool->getMetaMaxId(ns_id); break; } @@ -279,27 +287,23 @@ PageStorageRunMode StoragePool::restore() data_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, data_storage_v3); meta_storage_writer = std::make_shared(run_mode, /*storage_v2_*/ nullptr, meta_storage_v3); - max_log_page_id = global_storage_pool.getLogMaxId(ns_id); - max_data_page_id = global_storage_pool.getDataMaxId(ns_id); - max_meta_page_id = global_storage_pool.getMetaMaxId(ns_id); + max_log_page_id = global_storage_pool->getLogMaxId(ns_id); + max_data_page_id = global_storage_pool->getDataMaxId(ns_id); + max_meta_page_id = global_storage_pool->getMetaMaxId(ns_id); } else // Still running Mix Mode { - auto v2_log_max_ids = log_storage->restore(); - auto v2_data_max_ids = data_storage->restore(); - auto v2_meta_max_ids = meta_storage->restore(); + auto v2_log_max_ids = log_storage_v2->restore(); + auto v2_data_max_ids = data_storage_v2->restore(); + auto v2_meta_max_ids = meta_storage_v2->restore(); assert(v2_log_max_ids.size() == 1); assert(v2_data_max_ids.size() == 1); assert(v2_meta_max_ids.size() == 1); - max_log_page_id = v2_log_max_ids[0]; - max_data_page_id = v2_data_max_ids[0]; - max_meta_page_id = v2_meta_max_ids[0]; - - max_log_page_id = std::max(max_log_page_id, global_storage_pool.getLogMaxId(ns_id)); - max_data_page_id = std::max(max_data_page_id, global_storage_pool.getDataMaxId(ns_id)); - max_meta_page_id = std::max(max_meta_page_id, global_storage_pool.getMetaMaxId(ns_id)); + max_log_page_id = std::max(v2_log_max_ids[0], global_storage_pool->getLogMaxId(ns_id)); + max_data_page_id = std::max(v2_data_max_ids[0], global_storage_pool->getDataMaxId(ns_id)); + max_meta_page_id = std::max(v2_meta_max_ids[0], global_storage_pool->getMetaMaxId(ns_id)); } break; } diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index f75fb2fcad2..5f439899a84 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -51,24 +51,45 @@ class GlobalStoragePool : private boost::noncopyable friend class StoragePool; - std::map getLogMaxIds() const + PageId getLogMaxId(NamespaceId ns_id) const { - return log_max_ids; + PageId max_log_page_id = 0; + if (const auto & it = log_max_ids.find(ns_id); it != log_max_ids.end()) + { + max_log_page_id = it->second; + } + + return max_log_page_id; } - std::map getDataMaxIds() const + PageId getDataMaxId(NamespaceId ns_id) const { - return data_max_ids; + PageId max_data_page_id = 0; + if (const auto & it = data_max_ids.find(ns_id); it != data_max_ids.end()) + { + max_data_page_id = it->second; + } + + return max_data_page_id; } - std::map getMetaMaxIds() const + PageId getMetaMaxId(NamespaceId ns_id) const { - return meta_max_ids; + PageId max_meta_page_id = 0; + if (const auto & it = meta_max_ids.find(ns_id); it != meta_max_ids.end()) + { + max_meta_page_id = it->second; + } + + return max_meta_page_id; } + // GC immediately + // Only used on dbgFuncMisc + bool gc(); + private: - // TODO: maybe more frequent gc for GlobalStoragePool? - bool gc(const Settings & settings, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); + bool gc(const Settings & settings, bool immediately = false, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); private: PageStoragePtr log_storage; @@ -206,11 +227,6 @@ class StoragePool : private boost::noncopyable Context & global_context; - // TBD: Will be replaced GlobalPathPoolPtr after mix mode ptr ready - std::map v3_log_max_ids; - std::map v3_data_max_ids; - std::map v3_meta_max_ids; - std::atomic max_log_page_id = 0; std::atomic max_data_page_id = 0; std::atomic max_meta_page_id = 0; diff --git a/dbms/src/Storages/Page/PageStorage.cpp b/dbms/src/Storages/Page/PageStorage.cpp index 7f14d905858..fd81cd04a79 100644 --- a/dbms/src/Storages/Page/PageStorage.cpp +++ b/dbms/src/Storages/Page/PageStorage.cpp @@ -31,4 +31,506 @@ PageStoragePtr PageStorage::create( return std::make_shared(name, delegator, config, file_provider); } +/********************** + * PageReader methods * + *********************/ + +PageStorage::SnapshotPtr PageReader::toConcreteV3Snapshot() const +{ + return snap ? toConcreteMixedSnapshot(snap)->getV3Snasphot() : snap; +} + +PageStorage::SnapshotPtr PageReader::toConcreteV2Snapshot() const +{ + return snap ? toConcreteMixedSnapshot(snap)->getV2Snasphot() : snap; +} + +DB::Page PageReader::read(PageId page_id) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->read(ns_id, page_id, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->read(ns_id, page_id, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); + if (page_from_v3.isValid()) + { + return page_from_v3; + } + return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +PageMap PageReader::read(const PageIds & page_ids) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->read(ns_id, page_ids, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->read(ns_id, page_ids, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteV3Snapshot(), false); + PageIds invalid_page_ids; + for (const auto & [query_page_id, page] : page_maps) + { + if (!page.isValid()) + { + invalid_page_ids.emplace_back(query_page_id); + } + } + + if (!invalid_page_ids.empty()) + { + const auto & page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_ids, read_limiter, toConcreteV2Snapshot()); + for (const auto & [page_id_, page_] : page_maps_from_v2) + { + page_maps[page_id_] = page_; + } + } + + return page_maps; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + + +void PageReader::read(const PageIds & page_ids, PageHandler & handler) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->read(ns_id, page_ids, handler, read_limiter, snap); + break; + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->read(ns_id, page_ids, handler, read_limiter, snap); + break; + } + case PageStorageRunMode::MIX_MODE: + { + const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteV3Snapshot(), false); + storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteV2Snapshot()); + break; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +PageMap PageReader::read(const std::vector & page_fields) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->read(ns_id, page_fields, read_limiter, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->read(ns_id, page_fields, read_limiter, snap); + } + case PageStorageRunMode::MIX_MODE: + { + auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteV3Snapshot(), false); + + std::vector invalid_page_fields; + + for (const auto & page_field : page_fields) + { + if (!page_maps[page_field.first].isValid()) + { + invalid_page_fields.emplace_back(page_field); + } + } + + if (!invalid_page_fields.empty()) + { + auto page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_fields, read_limiter, toConcreteV2Snapshot()); + for (const auto & page_field_ : invalid_page_fields) + { + page_maps[page_field_.first] = page_maps_from_v2[page_field_.first]; + } + } + + return page_maps; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +PageId PageReader::getNormalPageId(PageId page_id) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getNormalPageId(ns_id, page_id, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getNormalPageId(ns_id, page_id, snap); + } + case PageStorageRunMode::MIX_MODE: + { + PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteV3Snapshot(), false); + if (resolved_page_id != INVALID_PAGE_ID) + { + return resolved_page_id; + } + return storage_v2->getNormalPageId(ns_id, page_id, toConcreteV2Snapshot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +UInt64 PageReader::getPageChecksum(PageId page_id) const +{ + return getPageEntry(page_id).checksum; +} + +PageEntry PageReader::getPageEntry(PageId page_id) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getEntry(ns_id, page_id, snap); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getEntry(ns_id, page_id, snap); + } + case PageStorageRunMode::MIX_MODE: + { + PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteV3Snapshot()); + if (page_entry.file_id != INVALID_BLOBFILE_ID) + { + return page_entry; + } + return storage_v2->getEntry(ns_id, page_id, toConcreteV2Snapshot()); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +PageStorage::SnapshotPtr PageReader::getSnapshot(const String & tracing_id) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getSnapshot(tracing_id); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getSnapshot(tracing_id); + } + case PageStorageRunMode::MIX_MODE: + { + return std::make_shared(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // + storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + + +SnapshotsStatistics PageReader::getSnapshotsStat() const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getSnapshotsStat(); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getSnapshotsStat(); + } + case PageStorageRunMode::MIX_MODE: + { + SnapshotsStatistics statistics_total; + const auto & statistics_from_v2 = storage_v2->getSnapshotsStat(); + const auto & statistics_from_v3 = storage_v3->getSnapshotsStat(); + + statistics_total.num_snapshots = statistics_from_v2.num_snapshots + statistics_from_v3.num_snapshots; + if (statistics_from_v2.longest_living_seconds > statistics_from_v3.longest_living_seconds) + { + statistics_total.longest_living_seconds = statistics_from_v2.longest_living_seconds; + statistics_total.longest_living_from_thread_id = statistics_from_v2.longest_living_from_thread_id; + statistics_total.longest_living_from_tracing_id = statistics_from_v2.longest_living_from_tracing_id; + } + else + { + statistics_total.longest_living_seconds = statistics_from_v3.longest_living_seconds; + statistics_total.longest_living_from_thread_id = statistics_from_v3.longest_living_from_thread_id; + statistics_total.longest_living_from_tracing_id = statistics_from_v3.longest_living_from_tracing_id; + } + + return statistics_total; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +void PageReader::traverse(const std::function & acceptor) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->traverse(acceptor, nullptr); + break; + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->traverse(acceptor, nullptr); + break; + } + case PageStorageRunMode::MIX_MODE: + { + // Used by RegionPersister::restore + // Must traverse storage_v3 before storage_v2 + storage_v3->traverse(acceptor, toConcreteV3Snapshot()); + storage_v2->traverse(acceptor, toConcreteV2Snapshot()); + break; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + + +/********************** + * PageWriter methods * + *********************/ + +void PageWriter::write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + writeIntoV2(std::move(write_batch), write_limiter); + break; + } + case PageStorageRunMode::ONLY_V3: + { + writeIntoV3(std::move(write_batch), write_limiter); + break; + } + case PageStorageRunMode::MIX_MODE: + { + writeIntoMixMode(std::move(write_batch), write_limiter); + break; + } + } +} + + +void PageWriter::writeIntoV2(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const +{ + storage_v2->write(std::move(write_batch), write_limiter); +} + +void PageWriter::writeIntoV3(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const +{ + storage_v3->write(std::move(write_batch), write_limiter); +} + +void PageWriter::writeIntoMixMode(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const +{ + const auto & ns_id = write_batch.getNamespaceId(); + WriteBatch wb_for_v2{ns_id}; + WriteBatch wb_for_put_v3{ns_id}; + + // If we do need copy entry from V2 into V3 + // We need hold mem from V2 pages after write. + std::list mem_holders; + + for (const auto & write : write_batch.getWrites()) + { + switch (write.type) + { + // PUT/PUT_EXTERNAL only for V3 + case WriteBatch::WriteType::PUT: + case WriteBatch::WriteType::PUT_EXTERNAL: + { + break; + } + // Both need del in v2 and v3 + case WriteBatch::WriteType::DEL: + { + wb_for_v2.copyWrite(write); + break; + } + case WriteBatch::WriteType::REF: + { + PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, + write.ori_page_id, + /*snapshot*/ nullptr, + false); + // If the normal id is not found in v3, read from v2 and create a new put + ref + if (resolved_page_id == INVALID_PAGE_ID) + { + const auto & entry_for_put = storage_v2->getEntry(ns_id, write.ori_page_id, /*snapshot*/ {}); + if (entry_for_put.file_id != 0) + { + auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); + + // Keep the mem hold, no need create new one. + mem_holders.emplace_back(page_for_put.mem_holder); + assert(entry_for_put.size == page_for_put.data.size()); + + // Page with fields + if (!entry_for_put.field_offsets.empty()) + { + wb_for_put_v3.putPage(write.ori_page_id, // + entry_for_put.tag, + std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), + page_for_put.data.size(), + Page::fieldOffsetsToSizes(entry_for_put.field_offsets, entry_for_put.size)); + } + else + { // Normal page with fields + wb_for_put_v3.putPage(write.ori_page_id, // + entry_for_put.tag, + std::make_shared(page_for_put.data.begin(), + page_for_put.data.size()), + page_for_put.data.size()); + } + + LOG_FMT_INFO(Logger::get("PageWriter"), "Can't find [origin_id={}] in v3. Created a new page with [field_offsets={}] into V3", write.ori_page_id, entry_for_put.field_offsets.size()); + } + else + { + throw Exception(fmt::format("Can't find origin entry in V2 and V3, [ns_id={}, ori_page_id={}]", + ns_id, + write.ori_page_id), + ErrorCodes::LOGICAL_ERROR); + } + } + // else V3 found the origin one. + // Then do nothing. + break; + } + default: + { + throw Exception(fmt::format("Unknown write type: {}", write.type)); + } + } + } + + if (!wb_for_put_v3.empty()) + { + // The `writes` in wb_for_put_v3 must come before the `writes` in write_batch + wb_for_put_v3.copyWrites(write_batch.getWrites()); + storage_v3->write(std::move(wb_for_put_v3), write_limiter); + } + else + { + storage_v3->write(std::move(write_batch), write_limiter); + } + + if (!wb_for_v2.empty()) + { + storage_v2->write(std::move(wb_for_v2), write_limiter); + } +} + + +PageStorage::Config PageWriter::getSettings() const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->getSettings(); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->getSettings(); + } + case PageStorageRunMode::MIX_MODE: + { + throw Exception("Not support.", ErrorCodes::NOT_IMPLEMENTED); + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + +void PageWriter::reloadSettings(const PageStorage::Config & new_config) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + storage_v2->reloadSettings(new_config); + break; + } + case PageStorageRunMode::ONLY_V3: + { + storage_v3->reloadSettings(new_config); + break; + } + case PageStorageRunMode::MIX_MODE: + { + storage_v2->reloadSettings(new_config); + storage_v3->reloadSettings(new_config); + break; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +}; + +bool PageWriter::gc(bool not_skip, const WriteLimiterPtr & write_limiter, const ReadLimiterPtr & read_limiter) const +{ + switch (run_mode) + { + case PageStorageRunMode::ONLY_V2: + { + return storage_v2->gc(not_skip, write_limiter, read_limiter); + } + case PageStorageRunMode::ONLY_V3: + { + return storage_v3->gc(not_skip, write_limiter, read_limiter); + } + case PageStorageRunMode::MIX_MODE: + { + bool ok = storage_v2->gc(not_skip, write_limiter, read_limiter); + ok |= storage_v3->gc(not_skip, write_limiter, read_limiter); + return ok; + } + default: + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + } +} + } // namespace DB diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index 7da3591ab2f..f9c87490ec3 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -374,296 +374,32 @@ class PageReader : private boost::noncopyable { } - DB::Page read(PageId page_id) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->read(ns_id, page_id, read_limiter, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->read(ns_id, page_id, read_limiter, snap); - } - case PageStorageRunMode::MIX_MODE: - { - const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); - if (page_from_v3.isValid()) - { - return page_from_v3; - } - return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + DB::Page read(PageId page_id) const; - PageMap read(const PageIds & page_ids) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->read(ns_id, page_ids, read_limiter, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->read(ns_id, page_ids, read_limiter, snap); - } - case PageStorageRunMode::MIX_MODE: - { - auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteV3Snapshot(), false); - PageIds invalid_page_ids; - for (const auto & [query_page_id, page] : page_maps) - { - if (!page.isValid()) - { - invalid_page_ids.emplace_back(query_page_id); - } - } - - if (!invalid_page_ids.empty()) - { - const auto & page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_ids, read_limiter, toConcreteV2Snapshot()); - for (const auto & [page_id_, page_] : page_maps_from_v2) - { - page_maps[page_id_] = page_; - } - } - - return page_maps; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + PageMap read(const PageIds & page_ids) const; - void read(const PageIds & page_ids, PageHandler & handler) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - storage_v2->read(ns_id, page_ids, handler, read_limiter, snap); - break; - } - case PageStorageRunMode::ONLY_V3: - { - storage_v3->read(ns_id, page_ids, handler, read_limiter, snap); - break; - } - case PageStorageRunMode::MIX_MODE: - { - const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteV3Snapshot(), false); - storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteV2Snapshot()); - break; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + void read(const PageIds & page_ids, PageHandler & handler) const; using PageReadFields = PageStorage::PageReadFields; - PageMap read(const std::vector & page_fields) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->read(ns_id, page_fields, read_limiter, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->read(ns_id, page_fields, read_limiter, snap); - } - case PageStorageRunMode::MIX_MODE: - { - auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteV3Snapshot(), false); - - std::vector invalid_page_fields; - - for (const auto & page_field : page_fields) - { - if (!page_maps[page_field.first].isValid()) - { - invalid_page_fields.emplace_back(page_field); - } - } - - if (!invalid_page_fields.empty()) - { - auto page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_fields, read_limiter, toConcreteV2Snapshot()); - for (const auto & page_field_ : invalid_page_fields) - { - page_maps[page_field_.first] = page_maps_from_v2[page_field_.first]; - } - } - - return page_maps; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + PageMap read(const std::vector & page_fields) const; - PageId getNormalPageId(PageId page_id) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getNormalPageId(ns_id, page_id, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getNormalPageId(ns_id, page_id, snap); - } - case PageStorageRunMode::MIX_MODE: - { - PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteV3Snapshot(), false); - if (resolved_page_id != INVALID_PAGE_ID) - { - return resolved_page_id; - } - return storage_v2->getNormalPageId(ns_id, page_id, toConcreteV2Snapshot()); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + PageId getNormalPageId(PageId page_id) const; - UInt64 getPageChecksum(PageId page_id) const - { - return getPageEntry(page_id).checksum; - } + UInt64 getPageChecksum(PageId page_id) const; - PageEntry getPageEntry(PageId page_id) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getEntry(ns_id, page_id, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getEntry(ns_id, page_id, snap); - } - case PageStorageRunMode::MIX_MODE: - { - PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteV3Snapshot()); - if (page_entry.file_id != INVALID_BLOBFILE_ID) - { - return page_entry; - } - return storage_v2->getEntry(ns_id, page_id, toConcreteV2Snapshot()); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + PageEntry getPageEntry(PageId page_id) const; - PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getSnapshot(tracing_id); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getSnapshot(tracing_id); - } - case PageStorageRunMode::MIX_MODE: - { - return std::make_shared(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // - storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const; // Get some statistics of all living snapshots and the oldest living snapshot. - SnapshotsStatistics getSnapshotsStat() const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getSnapshotsStat(); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getSnapshotsStat(); - } - case PageStorageRunMode::MIX_MODE: - { - SnapshotsStatistics statistics_total; - const auto & statistics_from_v2 = storage_v2->getSnapshotsStat(); - const auto & statistics_from_v3 = storage_v3->getSnapshotsStat(); - - statistics_total.num_snapshots = statistics_from_v2.num_snapshots + statistics_from_v3.num_snapshots; - if (statistics_from_v2.longest_living_seconds > statistics_from_v3.longest_living_seconds) - { - statistics_total.longest_living_seconds = statistics_from_v2.longest_living_seconds; - statistics_total.longest_living_from_thread_id = statistics_from_v2.longest_living_from_thread_id; - statistics_total.longest_living_from_tracing_id = statistics_from_v2.longest_living_from_tracing_id; - } - else - { - statistics_total.longest_living_seconds = statistics_from_v3.longest_living_seconds; - statistics_total.longest_living_from_thread_id = statistics_from_v3.longest_living_from_thread_id; - statistics_total.longest_living_from_tracing_id = statistics_from_v3.longest_living_from_tracing_id; - } - - return statistics_total; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + SnapshotsStatistics getSnapshotsStat() const; - void traverse(const std::function & acceptor) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - storage_v2->traverse(acceptor, nullptr); - break; - } - case PageStorageRunMode::ONLY_V3: - { - storage_v3->traverse(acceptor, nullptr); - break; - } - case PageStorageRunMode::MIX_MODE: - { - // Used by RegionPersister::restore - // Must traverse storage_v3 before storage_v2 - storage_v3->traverse(acceptor, toConcreteV3Snapshot()); - storage_v2->traverse(acceptor, toConcreteV2Snapshot()); - break; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + void traverse(const std::function & acceptor) const; private: - PageStorage::SnapshotPtr toConcreteV3Snapshot() const - { - return snap ? toConcreteMixedSnapshot(snap)->getV3Snasphot() : snap; - } + PageStorage::SnapshotPtr toConcreteV3Snapshot() const; - PageStorage::SnapshotPtr toConcreteV2Snapshot() const - { - return snap ? toConcreteMixedSnapshot(snap)->getV2Snasphot() : snap; - } + PageStorage::SnapshotPtr toConcreteV2Snapshot() const; private: const PageStorageRunMode run_mode; @@ -685,188 +421,25 @@ class PageWriter : private boost::noncopyable { } - void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - storage_v2->write(std::move(write_batch), write_limiter); - break; - } - case PageStorageRunMode::ONLY_V3: - { - storage_v3->write(std::move(write_batch), write_limiter); - break; - } - case PageStorageRunMode::MIX_MODE: - { - const auto & ns_id = write_batch.getNamespaceId(); - WriteBatch wb_for_v2{ns_id}; - WriteBatch wb_for_put_v3{ns_id}; - - for (const auto & write : write_batch.getWrites()) - { - switch (write.type) - { - // PUT/PUT_EXTERNAL only for V3 - case WriteBatch::WriteType::PUT: - case WriteBatch::WriteType::PUT_EXTERNAL: - { - break; - } - // Both need del in v2 and v3 - case WriteBatch::WriteType::DEL: - { - wb_for_v2.copyWrite(write); - break; - } - case WriteBatch::WriteType::REF: - { - PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, - write.ori_page_id, - /*snapshot*/ nullptr, - false); - // if normal id is not ok, read from v2 and create a new put + ref - if (resolved_page_id == INVALID_PAGE_ID) - { - const auto & entry_for_put = storage_v2->getEntry(ns_id, write.ori_page_id, /*snapshot*/ {}); - if (entry_for_put.file_id != 0) - { - auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); - assert(entry_for_put.size == page_for_put.data.size()); - - // Page with fields - if (!entry_for_put.field_offsets.empty()) - { - wb_for_put_v3.putPage(write.ori_page_id, // - 0, - std::make_shared(page_for_put.data.begin(), page_for_put.data.size()), - page_for_put.data.size(), - Page::fieldOffsetsToSizes(entry_for_put.field_offsets, entry_for_put.size)); - } - else - { // Normal page with fields - wb_for_put_v3.putPage(write.ori_page_id, // - 0, - std::make_shared(page_for_put.data.begin(), - page_for_put.data.size()), - page_for_put.data.size()); - } - - LOG_FMT_INFO(Logger::get("PageWriter"), "Can't find [origin_id={}] in v3. Created a new page with [field_offsets={}] into V3", write.ori_page_id, entry_for_put.field_offsets.size()); - } - else - { - throw Exception(fmt::format("Can't find origin entry in V2 and V3, [ns_id={}, ori_page_id={}]", - ns_id, - write.ori_page_id), - ErrorCodes::LOGICAL_ERROR); - } - } - // else V3 found the origin one. - // Then do nothing. - break; - } - default: - { - throw Exception(fmt::format("Unknown write type: {}", write.type)); - } - } - } - - if (!wb_for_put_v3.empty()) - { - // The `writes` in wb_for_put_v3 must come before the `writes` in write_batch - wb_for_put_v3.copyWrites(write_batch.getWrites()); - storage_v3->write(std::move(wb_for_put_v3), write_limiter); - } - else - { - storage_v3->write(std::move(write_batch), write_limiter); - } - - - if (!wb_for_v2.empty()) - { - storage_v2->write(std::move(wb_for_v2), write_limiter); - } - break; - } - } - } + void write(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const; friend class RegionPersister; private: - PageStorage::Config getSettings() const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getSettings(); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getSettings(); - } - case PageStorageRunMode::MIX_MODE: - { - throw Exception("Not support.", ErrorCodes::NOT_IMPLEMENTED); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + void writeIntoV2(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const; - void reloadSettings(const PageStorage::Config & new_config) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - storage_v2->reloadSettings(new_config); - break; - } - case PageStorageRunMode::ONLY_V3: - { - storage_v3->reloadSettings(new_config); - break; - } - case PageStorageRunMode::MIX_MODE: - { - storage_v2->reloadSettings(new_config); - storage_v3->reloadSettings(new_config); - break; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - }; + void writeIntoV3(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const; - bool gc(bool not_skip, const WriteLimiterPtr & write_limiter, const ReadLimiterPtr & read_limiter) const - { - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->gc(not_skip, write_limiter, read_limiter); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->gc(not_skip, write_limiter, read_limiter); - } - case PageStorageRunMode::MIX_MODE: - { - bool ok = storage_v2->gc(not_skip, write_limiter, read_limiter); - ok |= storage_v3->gc(not_skip, write_limiter, read_limiter); - return ok; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } - } + void writeIntoMixMode(WriteBatch && write_batch, WriteLimiterPtr write_limiter) const; + + // A wrap of getSettings only used for `RegionPersister::gc` + PageStorage::Config getSettings() const; + + // A wrap of reloadSettings only used for `RegionPersister::gc` + void reloadSettings(const PageStorage::Config & new_config) const; + + // A wrap of gc only used for `RegionPersister::gc` + bool gc(bool not_skip, const WriteLimiterPtr & write_limiter, const ReadLimiterPtr & read_limiter) const; private: PageStorageRunMode run_mode; From 3bd6ed6e4caa0ef2b3ff4f12c0e000150d543da6 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Wed, 11 May 2022 16:21:27 +0800 Subject: [PATCH 32/38] hide swich by inherite --- dbms/src/Storages/Page/PageStorage.cpp | 426 +++++++++++------- dbms/src/Storages/Page/PageStorage.h | 45 +- dbms/src/Storages/Page/Snapshot.h | 4 +- dbms/src/Storages/Page/V3/PageEntry.h | 18 +- .../V3/tests/gtest_page_storage_mix_mode.cpp | 2 +- 5 files changed, 281 insertions(+), 214 deletions(-) diff --git a/dbms/src/Storages/Page/PageStorage.cpp b/dbms/src/Storages/Page/PageStorage.cpp index fd81cd04a79..7e139c02aa8 100644 --- a/dbms/src/Storages/Page/PageStorage.cpp +++ b/dbms/src/Storages/Page/PageStorage.cpp @@ -31,59 +31,169 @@ PageStoragePtr PageStorage::create( return std::make_shared(name, delegator, config, file_provider); } -/********************** - * PageReader methods * - *********************/ +/*************************** + * PageReaderImpl methods * + **************************/ -PageStorage::SnapshotPtr PageReader::toConcreteV3Snapshot() const +class PageReaderImpl : private boost::noncopyable { - return snap ? toConcreteMixedSnapshot(snap)->getV3Snasphot() : snap; -} +public: + static std::unique_ptr create( + PageStorageRunMode run_mode_, + NamespaceId ns_id_, + PageStoragePtr storage_v2_, + PageStoragePtr storage_v3_, + const PageStorage::SnapshotPtr & snap_, + ReadLimiterPtr read_limiter_); -PageStorage::SnapshotPtr PageReader::toConcreteV2Snapshot() const -{ - return snap ? toConcreteMixedSnapshot(snap)->getV2Snasphot() : snap; -} + virtual ~PageReaderImpl() = default; -DB::Page PageReader::read(PageId page_id) const + virtual DB::Page read(PageId page_id) const = 0; + + virtual PageMap read(const PageIds & page_ids) const = 0; + + virtual void read(const PageIds & page_ids, PageHandler & handler) const = 0; + + using PageReadFields = PageStorage::PageReadFields; + virtual PageMap read(const std::vector & page_fields) const = 0; + + virtual PageId getNormalPageId(PageId page_id) const = 0; + + virtual UInt64 getPageChecksum(PageId page_id) const = 0; + + virtual PageEntry getPageEntry(PageId page_id) const = 0; + + virtual PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const = 0; + + // Get some statistics of all living snapshots and the oldest living snapshot. + virtual SnapshotsStatistics getSnapshotsStat() const = 0; + + virtual void traverse(const std::function & acceptor) const = 0; +}; + + +class PageReaderImplNormal : public PageReaderImpl { - switch (run_mode) +public: + /// Not snapshot read. + explicit PageReaderImplNormal(NamespaceId ns_id_, PageStoragePtr storage_, ReadLimiterPtr read_limiter_) + : ns_id(ns_id_) + , storage(storage_) + , read_limiter(read_limiter_) { - case PageStorageRunMode::ONLY_V2: + } + + /// Snapshot read. + PageReaderImplNormal(NamespaceId ns_id_, PageStoragePtr storage_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) + : ns_id(ns_id_) + , storage(storage_) + , snap(snap_) + , read_limiter(read_limiter_) { - return storage_v2->read(ns_id, page_id, read_limiter, snap); } - case PageStorageRunMode::ONLY_V3: + + DB::Page read(PageId page_id) const override { - return storage_v3->read(ns_id, page_id, read_limiter, snap); + return storage->read(ns_id, page_id, read_limiter, snap); } - case PageStorageRunMode::MIX_MODE: + + PageMap read(const PageIds & page_ids) const override { - const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); - if (page_from_v3.isValid()) - { - return page_from_v3; - } - return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); + return storage->read(ns_id, page_ids, read_limiter, snap); } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + + void read(const PageIds & page_ids, PageHandler & handler) const override + { + storage->read(ns_id, page_ids, handler, read_limiter, snap); } -} -PageMap PageReader::read(const PageIds & page_ids) const + using PageReadFields = PageStorage::PageReadFields; + PageMap read(const std::vector & page_fields) const override + { + return storage->read(ns_id, page_fields, read_limiter, snap); + } + + PageId getNormalPageId(PageId page_id) const override + { + return storage->getNormalPageId(ns_id, page_id, snap); + } + + UInt64 getPageChecksum(PageId page_id) const override + { + return getPageEntry(page_id).checksum; + } + + PageEntry getPageEntry(PageId page_id) const override + { + return storage->getEntry(ns_id, page_id, snap); + } + + PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const override + { + return storage->getSnapshot(tracing_id); + } + + // Get some statistics of all living snapshots and the oldest living snapshot. + SnapshotsStatistics getSnapshotsStat() const override + { + return storage->getSnapshotsStat(); + } + + void traverse(const std::function & acceptor) const override + { + storage->traverse(acceptor, nullptr); + } + +private: + NamespaceId ns_id; + PageStoragePtr storage; + PageStorage::SnapshotPtr snap; + ReadLimiterPtr read_limiter; +}; + + +class PageReaderImplMixed : public PageReaderImpl { - switch (run_mode) +public: + /// Not snapshot read. + explicit PageReaderImplMixed(NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, ReadLimiterPtr read_limiter_) + : ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) + , read_limiter(read_limiter_) { - case PageStorageRunMode::ONLY_V2: + } + + /// Snapshot read. + PageReaderImplMixed(NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) + : ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) + , snap(snap_) + , read_limiter(read_limiter_) { - return storage_v2->read(ns_id, page_ids, read_limiter, snap); } - case PageStorageRunMode::ONLY_V3: + + PageReaderImplMixed(NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr && snap_, ReadLimiterPtr read_limiter_) + : ns_id(ns_id_) + , storage_v2(storage_v2_) + , storage_v3(storage_v3_) + , snap(std::move(snap_)) + , read_limiter(read_limiter_) { - return storage_v3->read(ns_id, page_ids, read_limiter, snap); } - case PageStorageRunMode::MIX_MODE: + + DB::Page read(PageId page_id) const override + { + const auto & page_from_v3 = storage_v3->read(ns_id, page_id, read_limiter, toConcreteV3Snapshot(), false); + if (page_from_v3.isValid()) + { + return page_from_v3; + } + return storage_v2->read(ns_id, page_id, read_limiter, toConcreteV2Snapshot()); + } + + PageMap read(const PageIds & page_ids) const override { auto page_maps = storage_v3->read(ns_id, page_ids, read_limiter, toConcreteV3Snapshot(), false); PageIds invalid_page_ids; @@ -106,50 +216,15 @@ PageMap PageReader::read(const PageIds & page_ids) const return page_maps; } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } -} - -void PageReader::read(const PageIds & page_ids, PageHandler & handler) const -{ - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - storage_v2->read(ns_id, page_ids, handler, read_limiter, snap); - break; - } - case PageStorageRunMode::ONLY_V3: - { - storage_v3->read(ns_id, page_ids, handler, read_limiter, snap); - break; - } - case PageStorageRunMode::MIX_MODE: + void read(const PageIds & page_ids, PageHandler & handler) const override { const auto & page_ids_not_found = storage_v3->read(ns_id, page_ids, handler, read_limiter, toConcreteV3Snapshot(), false); storage_v2->read(ns_id, page_ids_not_found, handler, read_limiter, toConcreteV2Snapshot()); - break; - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } -} -PageMap PageReader::read(const std::vector & page_fields) const -{ - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->read(ns_id, page_fields, read_limiter, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->read(ns_id, page_fields, read_limiter, snap); - } - case PageStorageRunMode::MIX_MODE: + using PageReadFields = PageStorage::PageReadFields; + PageMap read(const std::vector & page_fields) const override { auto page_maps = storage_v3->read(ns_id, page_fields, read_limiter, toConcreteV3Snapshot(), false); @@ -166,32 +241,16 @@ PageMap PageReader::read(const std::vector & page_fields) const if (!invalid_page_fields.empty()) { auto page_maps_from_v2 = storage_v2->read(ns_id, invalid_page_fields, read_limiter, toConcreteV2Snapshot()); - for (const auto & page_field_ : invalid_page_fields) + for (const auto & page_field : invalid_page_fields) { - page_maps[page_field_.first] = page_maps_from_v2[page_field_.first]; + page_maps[page_field.first] = page_maps_from_v2[page_field.first]; } } return page_maps; } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } -} -PageId PageReader::getNormalPageId(PageId page_id) const -{ - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getNormalPageId(ns_id, page_id, snap); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getNormalPageId(ns_id, page_id, snap); - } - case PageStorageRunMode::MIX_MODE: + PageId getNormalPageId(PageId page_id) const override { PageId resolved_page_id = storage_v3->getNormalPageId(ns_id, page_id, toConcreteV3Snapshot(), false); if (resolved_page_id != INVALID_PAGE_ID) @@ -200,29 +259,13 @@ PageId PageReader::getNormalPageId(PageId page_id) const } return storage_v2->getNormalPageId(ns_id, page_id, toConcreteV2Snapshot()); } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } -} - -UInt64 PageReader::getPageChecksum(PageId page_id) const -{ - return getPageEntry(page_id).checksum; -} -PageEntry PageReader::getPageEntry(PageId page_id) const -{ - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: + UInt64 getPageChecksum(PageId page_id) const override { - return storage_v2->getEntry(ns_id, page_id, snap); + return getPageEntry(page_id).checksum; } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getEntry(ns_id, page_id, snap); - } - case PageStorageRunMode::MIX_MODE: + + PageEntry getPageEntry(PageId page_id) const override { PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteV3Snapshot()); if (page_entry.file_id != INVALID_BLOBFILE_ID) @@ -231,47 +274,16 @@ PageEntry PageReader::getPageEntry(PageId page_id) const } return storage_v2->getEntry(ns_id, page_id, toConcreteV2Snapshot()); } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } -} -PageStorage::SnapshotPtr PageReader::getSnapshot(const String & tracing_id) const -{ - switch (run_mode) + PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const override { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getSnapshot(tracing_id); + return std::make_shared( + storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), + storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getSnapshot(tracing_id); - } - case PageStorageRunMode::MIX_MODE: - { - return std::make_shared(storage_v2->getSnapshot(fmt::format("{}-v2", tracing_id)), // - storage_v3->getSnapshot(fmt::format("{}-v3", tracing_id))); - } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); - } -} - -SnapshotsStatistics PageReader::getSnapshotsStat() const -{ - switch (run_mode) - { - case PageStorageRunMode::ONLY_V2: - { - return storage_v2->getSnapshotsStat(); - } - case PageStorageRunMode::ONLY_V3: - { - return storage_v3->getSnapshotsStat(); - } - case PageStorageRunMode::MIX_MODE: + // Get some statistics of all living snapshots and the oldest living snapshot. + SnapshotsStatistics getSnapshotsStat() const override { SnapshotsStatistics statistics_total; const auto & statistics_from_v2 = storage_v2->getSnapshotsStat(); @@ -293,38 +305,128 @@ SnapshotsStatistics PageReader::getSnapshotsStat() const return statistics_total; } - default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + + void traverse(const std::function & acceptor) const override + { + // Used by RegionPersister::restore + // Must traverse storage_v3 before storage_v2 + storage_v3->traverse(acceptor, toConcreteV3Snapshot()); + storage_v2->traverse(acceptor, toConcreteV2Snapshot()); } -} -void PageReader::traverse(const std::function & acceptor) const +private: + PageStorage::SnapshotPtr toConcreteV3Snapshot() const + { + return snap ? toConcreteMixedSnapshot(snap)->getV3Snapshot() : snap; + } + + PageStorage::SnapshotPtr toConcreteV2Snapshot() const + { + return snap ? toConcreteMixedSnapshot(snap)->getV2Snapshot() : snap; + } + +private: + const NamespaceId ns_id; + PageStoragePtr storage_v2; + PageStoragePtr storage_v3; + PageStorage::SnapshotPtr snap; + ReadLimiterPtr read_limiter; +}; + +std::unique_ptr PageReaderImpl::create( + PageStorageRunMode run_mode_, + NamespaceId ns_id_, + PageStoragePtr storage_v2_, + PageStoragePtr storage_v3_, + const PageStorage::SnapshotPtr & snap_, + ReadLimiterPtr read_limiter_) { - switch (run_mode) + switch (run_mode_) { case PageStorageRunMode::ONLY_V2: { - storage_v2->traverse(acceptor, nullptr); - break; + return std::make_unique(ns_id_, storage_v2_, snap_, read_limiter_); } case PageStorageRunMode::ONLY_V3: { - storage_v3->traverse(acceptor, nullptr); - break; + return std::make_unique(ns_id_, storage_v3_, snap_, read_limiter_); } case PageStorageRunMode::MIX_MODE: { - // Used by RegionPersister::restore - // Must traverse storage_v3 before storage_v2 - storage_v3->traverse(acceptor, toConcreteV3Snapshot()); - storage_v2->traverse(acceptor, toConcreteV2Snapshot()); - break; + return std::make_unique(ns_id_, storage_v2_, storage_v3_, snap_, read_limiter_); } default: - throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); + throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode_)), ErrorCodes::LOGICAL_ERROR); } } +/*********************** + * PageReader methods * + **********************/ +/// Not snapshot read. +PageReader::PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, ReadLimiterPtr read_limiter_) + : impl(PageReaderImpl::create(run_mode_, ns_id_, storage_v2_, storage_v3_, /*snap_=*/nullptr, read_limiter_)) +{ +} + +/// Snapshot read. +PageReader::PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr snap_, ReadLimiterPtr read_limiter_) + : impl(PageReaderImpl::create(run_mode_, ns_id_, storage_v2_, storage_v3_, std::move(snap_), read_limiter_)) +{ +} + +PageReader::~PageReader() = default; + +DB::Page PageReader::read(PageId page_id) const +{ + return impl->read(page_id); +} + +PageMap PageReader::read(const PageIds & page_ids) const +{ + return impl->read(page_ids); +} + +void PageReader::read(const PageIds & page_ids, PageHandler & handler) const +{ + impl->read(page_ids, handler); +} + +PageMap PageReader::read(const std::vector & page_fields) const +{ + return impl->read(page_fields); +} + +PageId PageReader::getNormalPageId(PageId page_id) const +{ + return impl->getNormalPageId(page_id); +} + +UInt64 PageReader::getPageChecksum(PageId page_id) const +{ + return impl->getPageChecksum(page_id); +} + +PageEntry PageReader::getPageEntry(PageId page_id) const +{ + return impl->getPageEntry(page_id); +} + +PageStorage::SnapshotPtr PageReader::getSnapshot(const String & tracing_id) const +{ + return impl->getSnapshot(tracing_id); +} + +// Get some statistics of all living snapshots and the oldest living snapshot. +SnapshotsStatistics PageReader::getSnapshotsStat() const +{ + return impl->getSnapshotsStat(); +} + +void PageReader::traverse(const std::function & acceptor) const +{ + impl->traverse(acceptor); +} /********************** * PageWriter methods * diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index f9c87490ec3..c9f1d59a194 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -340,39 +341,19 @@ class PageStorage : private boost::noncopyable FileProviderPtr file_provider; }; +// An impl class to hide the details for PageReaderImplMixed +class PageReaderImpl; +// A class to wrap read with a specify snapshot class PageReader : private boost::noncopyable { public: /// Not snapshot read. - explicit PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, ReadLimiterPtr read_limiter_) - : run_mode(run_mode_) - , ns_id(ns_id_) - , storage_v2(storage_v2_) - , storage_v3(storage_v3_) - , read_limiter(read_limiter_) - { - } + explicit PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, ReadLimiterPtr read_limiter_); /// Snapshot read. - PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, const PageStorage::SnapshotPtr & snap_, ReadLimiterPtr read_limiter_) - : run_mode(run_mode_) - , ns_id(ns_id_) - , storage_v2(storage_v2_) - , storage_v3(storage_v3_) - , snap(snap_) - , read_limiter(read_limiter_) - { - } + PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr snap_, ReadLimiterPtr read_limiter_); - PageReader(const PageStorageRunMode & run_mode_, NamespaceId ns_id_, PageStoragePtr storage_v2_, PageStoragePtr storage_v3_, PageStorage::SnapshotPtr && snap_, ReadLimiterPtr read_limiter_) - : run_mode(run_mode_) - , ns_id(ns_id_) - , storage_v2(storage_v2_) - , storage_v3(storage_v3_) - , snap(std::move(snap_)) - , read_limiter(read_limiter_) - { - } + ~PageReader(); DB::Page read(PageId page_id) const; @@ -397,17 +378,7 @@ class PageReader : private boost::noncopyable void traverse(const std::function & acceptor) const; private: - PageStorage::SnapshotPtr toConcreteV3Snapshot() const; - - PageStorage::SnapshotPtr toConcreteV2Snapshot() const; - -private: - const PageStorageRunMode run_mode; - NamespaceId ns_id; - PageStoragePtr storage_v2; - PageStoragePtr storage_v3; - PageStorage::SnapshotPtr snap; - ReadLimiterPtr read_limiter; + std::unique_ptr impl; }; using PageReaderPtr = std::shared_ptr; diff --git a/dbms/src/Storages/Page/Snapshot.h b/dbms/src/Storages/Page/Snapshot.h index 622707e8fdf..77e68f1b054 100644 --- a/dbms/src/Storages/Page/Snapshot.h +++ b/dbms/src/Storages/Page/Snapshot.h @@ -45,12 +45,12 @@ class PageStorageSnapshotMixed : public PageStorageSnapshot ~PageStorageSnapshotMixed() = default; - PageStorageSnapshotPtr getV2Snasphot() + PageStorageSnapshotPtr getV2Snapshot() { return snapshot_v2; } - PageStorageSnapshotPtr getV3Snasphot() + PageStorageSnapshotPtr getV3Snapshot() { return snapshot_v3; } diff --git a/dbms/src/Storages/Page/V3/PageEntry.h b/dbms/src/Storages/Page/V3/PageEntry.h index 3e0dc0973df..e79397531a7 100644 --- a/dbms/src/Storages/Page/V3/PageEntry.h +++ b/dbms/src/Storages/Page/V3/PageEntry.h @@ -41,17 +41,6 @@ struct PageEntryV3 PageFieldOffsetChecksums field_offsets{}; public: - String toString() const - { - return fmt::format("[file_id={}],[offset={}][size={}],[tag={}],[crc=0x{:X}],[field_offsets size={}]", // - file_id, - offset, - size, - tag, - checksum, - field_offsets.size()); - } - PageSize getTotalSize() const { return size + padded_size; @@ -98,7 +87,12 @@ inline PageIdV3Internal buildV3Id(NamespaceId n_id, PageId p_id) inline String toDebugString(const PageEntryV3 & entry) { - return fmt::format("PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}}}", entry.file_id, entry.offset, entry.size, entry.checksum); + return fmt::format("PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets_size: {}}}", + entry.file_id, + entry.offset, + entry.size, + entry.checksum, + entry.tag, entry.field_offsets.size()); } } // namespace PS::V3 diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index d60ca432f02..d3fdafe57e8 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -461,7 +461,7 @@ try ASSERT_EQ(page_reader_mix->getNormalPageId(10), 8); std::vector read_fields; - read_fields.emplace_back(std::make_pair(10, {0, 1, 2, 6})); + read_fields.emplace_back(std::pair(10, {0, 1, 2, 6})); PageMap page_maps = page_reader_mix->read(read_fields); ASSERT_EQ(page_maps.size(), 1); From 53904fc24926257bee3fdbc72d32052def7666b7 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Wed, 11 May 2022 16:31:33 +0800 Subject: [PATCH 33/38] Remove useless method --- dbms/src/Storages/Page/PageStorage.cpp | 17 ----------------- dbms/src/Storages/Page/PageStorage.h | 2 -- 2 files changed, 19 deletions(-) diff --git a/dbms/src/Storages/Page/PageStorage.cpp b/dbms/src/Storages/Page/PageStorage.cpp index 7e139c02aa8..244e62b845c 100644 --- a/dbms/src/Storages/Page/PageStorage.cpp +++ b/dbms/src/Storages/Page/PageStorage.cpp @@ -59,8 +59,6 @@ class PageReaderImpl : private boost::noncopyable virtual PageId getNormalPageId(PageId page_id) const = 0; - virtual UInt64 getPageChecksum(PageId page_id) const = 0; - virtual PageEntry getPageEntry(PageId page_id) const = 0; virtual PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const = 0; @@ -118,11 +116,6 @@ class PageReaderImplNormal : public PageReaderImpl return storage->getNormalPageId(ns_id, page_id, snap); } - UInt64 getPageChecksum(PageId page_id) const override - { - return getPageEntry(page_id).checksum; - } - PageEntry getPageEntry(PageId page_id) const override { return storage->getEntry(ns_id, page_id, snap); @@ -260,11 +253,6 @@ class PageReaderImplMixed : public PageReaderImpl return storage_v2->getNormalPageId(ns_id, page_id, toConcreteV2Snapshot()); } - UInt64 getPageChecksum(PageId page_id) const override - { - return getPageEntry(page_id).checksum; - } - PageEntry getPageEntry(PageId page_id) const override { PageEntry page_entry = storage_v3->getEntry(ns_id, page_id, toConcreteV3Snapshot()); @@ -402,11 +390,6 @@ PageId PageReader::getNormalPageId(PageId page_id) const return impl->getNormalPageId(page_id); } -UInt64 PageReader::getPageChecksum(PageId page_id) const -{ - return impl->getPageChecksum(page_id); -} - PageEntry PageReader::getPageEntry(PageId page_id) const { return impl->getPageEntry(page_id); diff --git a/dbms/src/Storages/Page/PageStorage.h b/dbms/src/Storages/Page/PageStorage.h index c9f1d59a194..fe8946549cd 100644 --- a/dbms/src/Storages/Page/PageStorage.h +++ b/dbms/src/Storages/Page/PageStorage.h @@ -366,8 +366,6 @@ class PageReader : private boost::noncopyable PageId getNormalPageId(PageId page_id) const; - UInt64 getPageChecksum(PageId page_id) const; - PageEntry getPageEntry(PageId page_id) const; PageStorage::SnapshotPtr getSnapshot(const String & tracing_id) const; From 2721ca4f6b6b4be0efb3bd6f721cd3ac3280966b Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 11 May 2022 16:42:25 +0800 Subject: [PATCH 34/38] fix restore not right --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 90a9d578e15..701621d584a 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -269,6 +269,14 @@ PageStorageRunMode StoragePool::restore() } case PageStorageRunMode::MIX_MODE: { + auto v2_log_max_ids = log_storage_v2->restore(); + auto v2_data_max_ids = data_storage_v2->restore(); + auto v2_meta_max_ids = meta_storage_v2->restore(); + + assert(v2_log_max_ids.size() == 1); + assert(v2_data_max_ids.size() == 1); + assert(v2_meta_max_ids.size() == 1); + // Check number of valid pages in v2 // If V2 already have no any data in disk, Then change run_mode to ONLY_V3 if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) @@ -293,14 +301,6 @@ PageStorageRunMode StoragePool::restore() } else // Still running Mix Mode { - auto v2_log_max_ids = log_storage_v2->restore(); - auto v2_data_max_ids = data_storage_v2->restore(); - auto v2_meta_max_ids = meta_storage_v2->restore(); - - assert(v2_log_max_ids.size() == 1); - assert(v2_data_max_ids.size() == 1); - assert(v2_meta_max_ids.size() == 1); - max_log_page_id = std::max(v2_log_max_ids[0], global_storage_pool->getLogMaxId(ns_id)); max_data_page_id = std::max(v2_data_max_ids[0], global_storage_pool->getDataMaxId(ns_id)); max_meta_page_id = std::max(v2_meta_max_ids[0], global_storage_pool->getMetaMaxId(ns_id)); From 5a73c8660e47328e938b2f5355fd51ad0209f803 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 11 May 2022 17:09:29 +0800 Subject: [PATCH 35/38] update --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 4 ++- dbms/src/Storages/Page/V3/BlobStore.cpp | 22 ++++++++++++++ dbms/src/Storages/Page/V3/PageStorageImpl.cpp | 30 +++---------------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 701621d584a..2d0848ce5b3 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -282,7 +282,7 @@ PageStorageRunMode StoragePool::restore() if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) { LOG_FMT_INFO(logger, "Current pagestorage change from {} to {}", static_cast(PageStorageRunMode::MIX_MODE), static_cast(PageStorageRunMode::ONLY_V3)); - run_mode = PageStorageRunMode::ONLY_V3; + log_storage_v2 = nullptr; data_storage_v2 = nullptr; meta_storage_v2 = nullptr; @@ -298,6 +298,8 @@ PageStorageRunMode StoragePool::restore() max_log_page_id = global_storage_pool->getLogMaxId(ns_id); max_data_page_id = global_storage_pool->getDataMaxId(ns_id); max_meta_page_id = global_storage_pool->getMetaMaxId(ns_id); + + run_mode = PageStorageRunMode::ONLY_V3; } else // Still running Mix Mode { diff --git a/dbms/src/Storages/Page/V3/BlobStore.cpp b/dbms/src/Storages/Page/V3/BlobStore.cpp index 2c568d8e85b..3e58a3eb87e 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.cpp +++ b/dbms/src/Storages/Page/V3/BlobStore.cpp @@ -380,6 +380,11 @@ void BlobStore::removePosFromStats(BlobFileId blob_id, BlobFileOffset offset, si void BlobStore::read(PageIDAndEntriesV3 & entries, const PageHandler & handler, const ReadLimiterPtr & read_limiter) { + if (entries.empty()) + { + return; + } + ProfileEvents::increment(ProfileEvents::PSMReadPages, entries.size()); // Sort in ascending order by offset in file. @@ -444,6 +449,11 @@ void BlobStore::read(PageIDAndEntriesV3 & entries, const PageHandler & handler, PageMap BlobStore::read(FieldReadInfos & to_read, const ReadLimiterPtr & read_limiter) { + if (to_read.empty()) + { + return {}; + } + ProfileEvents::increment(ProfileEvents::PSMReadPages, to_read.size()); // Sort in ascending order by offset in file. @@ -542,6 +552,11 @@ PageMap BlobStore::read(FieldReadInfos & to_read, const ReadLimiterPtr & read_li PageMap BlobStore::read(PageIDAndEntriesV3 & entries, const ReadLimiterPtr & read_limiter) { + if (entries.empty()) + { + return {}; + } + ProfileEvents::increment(ProfileEvents::PSMReadPages, entries.size()); // Sort in ascending order by offset in file. @@ -621,6 +636,13 @@ PageMap BlobStore::read(PageIDAndEntriesV3 & entries, const ReadLimiterPtr & rea Page BlobStore::read(const PageIDAndEntryV3 & id_entry, const ReadLimiterPtr & read_limiter) { + if (!id_entry.second.isValid()) + { + Page page_not_found; + page_not_found.page_id = INVALID_PAGE_ID; + return page_not_found; + } + const auto & [page_id_v3, entry] = id_entry; const size_t buf_size = entry.size; diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp index edf9cb0c9db..de0a26f68c7 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.cpp +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.cpp @@ -139,13 +139,6 @@ DB::Page PageStorageImpl::readImpl(NamespaceId ns_id, PageId page_id, const Read } auto page_entry = throw_on_not_exist ? page_directory->get(buildV3Id(ns_id, page_id), snapshot) : page_directory->getOrNull(buildV3Id(ns_id, page_id), snapshot); - if (!page_entry.second.isValid()) - { - Page page_not_found; - page_not_found.page_id = INVALID_PAGE_ID; - return page_not_found; - } - return blob_store.read(page_entry, read_limiter); } @@ -162,7 +155,6 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c page_id_v3s.emplace_back(buildV3Id(ns_id, p_id)); } - if (throw_on_not_exist) { auto page_entries = page_directory->get(page_id_v3s, snapshot); @@ -171,11 +163,7 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c else { auto [page_entries, page_ids_not_found] = page_directory->getOrNull(page_id_v3s, snapshot); - PageMap page_map = {}; - if (!page_entries.empty()) - { - page_map = blob_store.read(page_entries, read_limiter); - } + PageMap page_map = blob_store.read(page_entries, read_limiter); for (const auto & page_id_not_found : page_ids_not_found) { @@ -201,19 +189,13 @@ PageIds PageStorageImpl::readImpl(NamespaceId ns_id, const PageIds & page_ids, c if (throw_on_not_exist) { auto page_entries = page_directory->get(page_id_v3s, snapshot); - if (!page_entries.empty()) - { - blob_store.read(page_entries, handler, read_limiter); - } + blob_store.read(page_entries, handler, read_limiter); return {}; } else { auto [page_entries, page_ids_not_found] = page_directory->getOrNull(page_id_v3s, snapshot); - if (!page_entries.empty()) - { - blob_store.read(page_entries, handler, read_limiter); - } + blob_store.read(page_entries, handler, read_limiter); return page_ids_not_found; } } @@ -241,11 +223,7 @@ PageMap PageStorageImpl::readImpl(NamespaceId ns_id, const std::vector Date: Wed, 11 May 2022 17:11:52 +0800 Subject: [PATCH 36/38] fix format --- dbms/src/Storages/Page/V3/PageEntry.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Page/V3/PageEntry.h b/dbms/src/Storages/Page/V3/PageEntry.h index e79397531a7..bcefc0e845b 100644 --- a/dbms/src/Storages/Page/V3/PageEntry.h +++ b/dbms/src/Storages/Page/V3/PageEntry.h @@ -92,7 +92,8 @@ inline String toDebugString(const PageEntryV3 & entry) entry.offset, entry.size, entry.checksum, - entry.tag, entry.field_offsets.size()); + entry.tag, + entry.field_offsets.size()); } } // namespace PS::V3 From b66007d331194d5c69e40024e1348c42c41387d4 Mon Sep 17 00:00:00 2001 From: JaySon Date: Wed, 11 May 2022 17:15:12 +0800 Subject: [PATCH 37/38] Apply suggestions from code review --- dbms/src/Storages/Page/PageStorage.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Page/PageStorage.cpp b/dbms/src/Storages/Page/PageStorage.cpp index 244e62b845c..cf8a9698a55 100644 --- a/dbms/src/Storages/Page/PageStorage.cpp +++ b/dbms/src/Storages/Page/PageStorage.cpp @@ -484,7 +484,7 @@ void PageWriter::writeIntoMixMode(WriteBatch && write_batch, WriteLimiterPtr wri if (resolved_page_id == INVALID_PAGE_ID) { const auto & entry_for_put = storage_v2->getEntry(ns_id, write.ori_page_id, /*snapshot*/ {}); - if (entry_for_put.file_id != 0) + if (entry_for_put.isValid()) { auto page_for_put = storage_v2->read(ns_id, write.ori_page_id); @@ -510,7 +510,12 @@ void PageWriter::writeIntoMixMode(WriteBatch && write_batch, WriteLimiterPtr wri page_for_put.data.size()); } - LOG_FMT_INFO(Logger::get("PageWriter"), "Can't find [origin_id={}] in v3. Created a new page with [field_offsets={}] into V3", write.ori_page_id, entry_for_put.field_offsets.size()); + LOG_FMT_INFO( + Logger::get("PageWriter"), + "Can't find the origin page in v3, migrate a new being ref page into V3 [page_id={}] [origin_id={}] [field_offsets={}]", + write.page_id, + write.ori_page_id, + entry_for_put.field_offsets.size()); } else { From 93793bde2ade24bd94d4c9e2c238a9974bc6736e Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Thu, 12 May 2022 09:04:58 +0800 Subject: [PATCH 38/38] update --- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 18 +++++++++++------- .../Page/V3/tests/gtest_page_storage.cpp | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 2d0848ce5b3..2979a75e192 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -281,6 +281,10 @@ PageStorageRunMode StoragePool::restore() // If V2 already have no any data in disk, Then change run_mode to ONLY_V3 if (log_storage_v2->getNumberOfPages() == 0 && data_storage_v2->getNumberOfPages() == 0 && meta_storage_v2->getNumberOfPages() == 0) { + // TODO: when `compaction` happend + // 1. Will rewrite meta into V3 by DT. + // 2. Need `DEL` meta in V2. + // 3. Need drop V2 in disk and check. LOG_FMT_INFO(logger, "Current pagestorage change from {} to {}", static_cast(PageStorageRunMode::MIX_MODE), static_cast(PageStorageRunMode::ONLY_V3)); log_storage_v2 = nullptr; @@ -312,13 +316,13 @@ PageStorageRunMode StoragePool::restore() default: throw Exception(fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), ErrorCodes::LOGICAL_ERROR); } - LOG_FMT_WARNING(logger, "Finished StoragePool restore. [current_run_mode={}] [ns_id={}]" - " [max_log_page_id={}] [max_data_page_id={}] [max_meta_page_id={}]", - static_cast(run_mode), - ns_id, - max_log_page_id, - max_data_page_id, - max_meta_page_id); + LOG_FMT_TRACE(logger, "Finished StoragePool restore. [current_run_mode={}] [ns_id={}]" + " [max_log_page_id={}] [max_data_page_id={}] [max_meta_page_id={}]", + static_cast(run_mode), + ns_id, + max_log_page_id, + max_data_page_id, + max_meta_page_id); return run_mode; } diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp index a1cf73dd10a..4bd3b2832b0 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp @@ -190,7 +190,7 @@ try rate_target, LimiterType::UNKNOW); - std::vector page_ids; + PageIds page_ids; for (size_t i = 0; i < wb_nums; ++i) { page_ids.emplace_back(page_id + i);