From 4954a24c0f15f9327fff1591778cd61c64b8e8dd Mon Sep 17 00:00:00 2001 From: lidezhu <47731263+lidezhu@users.noreply.github.com> Date: Tue, 1 Mar 2022 14:35:45 +0800 Subject: [PATCH] separate PageIdGenerator from StoragePool (#4154) ref pingcap/tics#3594 --- dbms/src/Server/DTTool/DTToolBench.cpp | 20 ++--- dbms/src/Server/tests/gtest_dttool.cpp | 2 + .../DeltaMerge/ColumnFile/ColumnFileTiny.cpp | 2 +- dbms/src/Storages/DeltaMerge/DMContext.h | 4 + .../Delta/ColumnFilePersistedSet.cpp | 4 +- .../Storages/DeltaMerge/Delta/MemTableSet.cpp | 4 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 10 ++- .../src/Storages/DeltaMerge/DeltaMergeStore.h | 1 + dbms/src/Storages/DeltaMerge/Segment.cpp | 22 +++--- dbms/src/Storages/DeltaMerge/StoragePool.cpp | 75 ++++++++++--------- dbms/src/Storages/DeltaMerge/StoragePool.h | 35 +++++---- .../tests/gtest_dm_delta_value_space.cpp | 6 +- .../DeltaMerge/tests/gtest_dm_file.cpp | 6 ++ .../DeltaMerge/tests/gtest_dm_segment.cpp | 8 +- .../tests/gtest_dm_segment_common_handle.cpp | 6 +- 15 files changed, 122 insertions(+), 83 deletions(-) diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index cbf86407f2d..b7eaaed96db 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -218,31 +218,31 @@ int benchEntry(const std::vector & opts) std::cerr << "invalid dtfile version: " << version << std::endl; return -EINVAL; } - auto algorithm_ = vm["algorithm"].as(); + auto algorithm_config = vm["algorithm"].as(); DB::ChecksumAlgo algorithm; - if (algorithm_ == "xxh3") + if (algorithm_config == "xxh3") { algorithm = DB::ChecksumAlgo::XXH3; } - else if (algorithm_ == "crc32") + else if (algorithm_config == "crc32") { algorithm = DB::ChecksumAlgo::CRC32; } - else if (algorithm_ == "crc64") + else if (algorithm_config == "crc64") { algorithm = DB::ChecksumAlgo::CRC64; } - else if (algorithm_ == "city128") + else if (algorithm_config == "city128") { algorithm = DB::ChecksumAlgo::City128; } - else if (algorithm_ == "none") + else if (algorithm_config == "none") { algorithm = DB::ChecksumAlgo::None; } else { - std::cerr << "invalid algorithm: " << algorithm_ << std::endl; + std::cerr << "invalid algorithm: " << algorithm_config << std::endl; return -EINVAL; } auto frame = vm["frame"].as(); @@ -286,7 +286,7 @@ int benchEntry(const std::vector & opts) "encryption: {}\n" "algorithm: {}"; DB::DM::DMConfigurationOpt opt = std::nullopt; - auto logger = &Poco::Logger::get("DTTool::Bench"); + auto * logger = &Poco::Logger::get("DTTool::Bench"); if (version == 1) { LOG_FMT_INFO(logger, SUMMARY_TEMPLATE_V1, version, column, size, field, random, encryption, workdir); @@ -294,7 +294,7 @@ int benchEntry(const std::vector & opts) } else { - LOG_FMT_INFO(logger, SUMMARY_TEMPLATE_V2, version, column, size, field, random, workdir, frame, encryption, algorithm_); + LOG_FMT_INFO(logger, SUMMARY_TEMPLATE_V2, version, column, size, field, random, workdir, frame, encryption, algorithm_config); opt.emplace(std::map{}, frame, algorithm); DB::STORAGE_FORMAT_CURRENT = DB::STORAGE_FORMAT_V3; } @@ -323,11 +323,13 @@ int benchEntry(const std::vector & opts) auto db_context = env.getContext(); auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); auto storage_pool = std::make_unique("test.t1", *path_pool, *db_context, db_context->getSettingsRef()); + auto page_id_generator = std::make_unique(); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, *path_pool, *storage_pool, + *page_id_generator, /*hash_salt*/ 0, 0, dm_settings.not_compress_columns, diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index bb1e2c1daa9..0004d38eba8 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -58,11 +58,13 @@ struct DTToolTest : public DB::base::TiFlashStorageTestBasic } auto path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); auto storage_pool = std::make_unique("test.t1", *path_pool, *db_context, db_context->getSettingsRef()); + auto page_id_generator = std::make_unique(); auto dm_settings = DB::DM::DeltaMergeStore::Settings{}; auto dm_context = std::make_unique( // *db_context, *path_pool, *storage_pool, + *page_id_generator, /*hash_salt*/ 0, 0, dm_settings.not_compress_columns, diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp index 3a4adf37963..75a3ef6e528 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp @@ -187,7 +187,7 @@ ColumnTinyFilePtr ColumnFileTiny::writeColumnFile(DMContext & context, const Blo PageId ColumnFileTiny::writeColumnFileData(DMContext & context, const Block & block, size_t offset, size_t limit, WriteBatches & wbs) { - auto page_id = context.storage_pool.newLogPageId(); + auto page_id = context.page_id_generator.newLogPageId(); MemoryWriteBuffer write_buf; PageFieldSizes col_data_sizes; diff --git a/dbms/src/Storages/DeltaMerge/DMContext.h b/dbms/src/Storages/DeltaMerge/DMContext.h index 24eb0ed18be..47edc5fab71 100644 --- a/dbms/src/Storages/DeltaMerge/DMContext.h +++ b/dbms/src/Storages/DeltaMerge/DMContext.h @@ -13,6 +13,7 @@ class StoragePathPool; namespace DM { class StoragePool; +class PageIdGenerator; using NotCompress = std::unordered_set; struct DMContext; using DMContextPtr = std::shared_ptr; @@ -26,6 +27,7 @@ struct DMContext : private boost::noncopyable StoragePathPool & path_pool; StoragePool & storage_pool; + PageIdGenerator & page_id_generator; const UInt64 hash_salt; // gc safe-point, maybe update. @@ -73,6 +75,7 @@ struct DMContext : private boost::noncopyable DMContext(const Context & db_context_, StoragePathPool & path_pool_, StoragePool & storage_pool_, + PageIdGenerator & page_id_generator_, const UInt64 hash_salt_, const DB::Timestamp min_version_, const NotCompress & not_compress_, @@ -83,6 +86,7 @@ struct DMContext : private boost::noncopyable : db_context(db_context_) , path_pool(path_pool_) , storage_pool(storage_pool_) + , page_id_generator(page_id_generator_) , hash_salt(hash_salt_) , min_version(min_version_) , not_compress(not_compress_) diff --git a/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp b/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp index acd76319900..e23532e3604 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp @@ -193,7 +193,7 @@ ColumnFilePersisteds ColumnFilePersistedSet::checkHeadAndCloneTail(DMContext & c else if (auto * t_file = column_file->tryToTinyFile(); t_file) { // Use a newly created page_id to reference the data page_id of current column file. - PageId new_data_page_id = context.storage_pool.newLogPageId(); + PageId new_data_page_id = context.page_id_generator.newLogPageId(); wbs.log.putRefPage(new_data_page_id, t_file->getDataPageId()); auto new_column_file = t_file->cloneWith(new_data_page_id); cloned_tail.push_back(new_column_file); @@ -201,7 +201,7 @@ ColumnFilePersisteds ColumnFilePersistedSet::checkHeadAndCloneTail(DMContext & c else if (auto * b_file = column_file->tryToBigFile(); b_file) { auto delegator = context.path_pool.getStableDiskDelegator(); - auto new_ref_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + auto new_ref_id = context.page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); auto file_id = b_file->getFile()->fileId(); wbs.data.putRefPage(new_ref_id, file_id); auto file_parent_path = delegator.getDTFilePath(file_id); diff --git a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp index 2881f13fbf2..19b5a69d58f 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp @@ -78,7 +78,7 @@ ColumnFiles MemTableSet::cloneColumnFiles(DMContext & context, const RowKeyRange else if (auto * t = column_file->tryToTinyFile(); t) { // Use a newly created page_id to reference the data page_id of current column file. - PageId new_data_page_id = context.storage_pool.newLogPageId(); + PageId new_data_page_id = context.page_id_generator.newLogPageId(); wbs.log.putRefPage(new_data_page_id, t->getDataPageId()); auto new_column_file = t->cloneWith(new_data_page_id); @@ -87,7 +87,7 @@ ColumnFiles MemTableSet::cloneColumnFiles(DMContext & context, const RowKeyRange else if (auto * f = column_file->tryToBigFile(); f) { auto delegator = context.path_pool.getStableDiskDelegator(); - auto new_ref_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + auto new_ref_id = context.page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); auto file_id = f->getFile()->fileId(); wbs.data.putRefPage(new_ref_id, file_id); auto file_parent_path = delegator.getDTFilePath(file_id); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 5a5f315c87a..d58e5fc4dd1 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -217,10 +217,11 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, try { storage_pool.restore(); // restore from disk - if (!storage_pool.maxMetaPageId()) + page_id_generator.restore(storage_pool); + if (!page_id_generator.maxMetaPageId()) { // Create the first segment. - auto segment_id = storage_pool.newMetaPageId(); + auto segment_id = page_id_generator.newMetaPageId(); if (segment_id != DELTA_MERGE_FIRST_SEGMENT_ID) throw Exception(fmt::format("The first segment id should be {}", DELTA_MERGE_FIRST_SEGMENT_ID), ErrorCodes::LOGICAL_ERROR); auto first_segment @@ -391,6 +392,7 @@ DMContextPtr DeltaMergeStore::newDMContext(const Context & db_context, const DB: auto * ctx = new DMContext(db_context.getGlobalContext(), path_pool, storage_pool, + page_id_generator, hash_salt, latest_gc_safe_point.load(std::memory_order_acquire), settings.not_compress_columns, @@ -591,7 +593,7 @@ std::tuple DeltaMergeStore::preAllocateIngestFile() auto delegator = path_pool.getStableDiskDelegator(); auto parent_path = delegator.choosePath(); - auto new_id = storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + auto new_id = page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); return {parent_path, new_id}; } @@ -708,7 +710,7 @@ void DeltaMergeStore::ingestFiles( /// Generate DMFile instance with a new ref_id pointed to the file_id. auto file_id = file->fileId(); const auto & file_parent_path = file->parentPath(); - auto ref_id = storage_pool.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); + auto ref_id = page_id_generator.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); auto ref_file = DMFile::restore(file_provider, file_id, ref_id, file_parent_path, DMFile::ReadMetaMode::all()); auto column_file = std::make_shared(*dm_context, ref_file, segment_range); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 6dc8acdfba7..71ed4c87cbb 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -432,6 +432,7 @@ class DeltaMergeStore : private boost::noncopyable StoragePathPool path_pool; Settings settings; StoragePool storage_pool; + PageIdGenerator page_id_generator; String db_name; String table_name; diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 0d92e81bbf5..86b2cb75aaa 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -145,7 +145,7 @@ StableValueSpacePtr createNewStable(DMContext & context, DMFileBlockOutputStream::Flags flags; flags.setSingleFile(context.db_context.getSettingsRef().dt_enable_single_file_mode_dmfile); - PageId dtfile_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + PageId dtfile_id = context.page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); auto dtfile = writeIntoNewDMFile(context, schema_snap, input_stream, dtfile_id, store_path, flags); auto stable = std::make_shared(stable_id); stable->setFiles({dtfile}, RowKeyRange::newAll(context.is_common_handle, context.rowkey_column_size)); @@ -215,8 +215,8 @@ SegmentPtr Segment::newSegment( rowkey_range, segment_id, next_segment_id, - context.storage_pool.newMetaPageId(), - context.storage_pool.newMetaPageId()); + context.page_id_generator.newMetaPageId(), + context.page_id_generator.newMetaPageId()); } SegmentPtr Segment::restoreSegment(DMContext & context, PageId segment_id) @@ -882,7 +882,7 @@ std::optional Segment::prepareSplitLogical(DMContext & dm_co EventRecorder recorder(ProfileEvents::DMSegmentSplit, ProfileEvents::DMSegmentSplitNS); - auto & storage_pool = dm_context.storage_pool; + auto & page_id_generator = dm_context.page_id_generator; RowKeyRange my_range(rowkey_range.start, split_point, is_common_handle, rowkey_column_size); RowKeyRange other_range(split_point, rowkey_range.end, is_common_handle, rowkey_column_size); @@ -899,7 +899,7 @@ std::optional Segment::prepareSplitLogical(DMContext & dm_co } GenPageId log_gen_page_id = [&]() { - return storage_pool.newLogPageId(); + return page_id_generator.newLogPageId(); }; DMFiles my_stable_files; @@ -912,8 +912,8 @@ std::optional Segment::prepareSplitLogical(DMContext & dm_co auto file_id = dmfile->fileId(); auto file_parent_path = delegate.getDTFilePath(file_id); - auto my_dmfile_id = storage_pool.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); - auto other_dmfile_id = storage_pool.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); + auto my_dmfile_id = page_id_generator.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); + auto other_dmfile_id = page_id_generator.newDataPageIdForDTFile(delegate, __PRETTY_FUNCTION__); wbs.data.putRefPage(my_dmfile_id, file_id); wbs.data.putRefPage(other_dmfile_id, file_id); @@ -936,7 +936,7 @@ std::optional Segment::prepareSplitLogical(DMContext & dm_co other_stable_files.push_back(other_dmfile); } - auto other_stable_id = storage_pool.newMetaPageId(); + auto other_stable_id = page_id_generator.newMetaPageId(); auto my_stable = std::make_shared(segment_snap->stable->getId()); auto other_stable = std::make_shared(other_stable_id); @@ -1038,7 +1038,7 @@ std::optional Segment::prepareSplitPhysical(DMContext & dm_c *read_info.read_columns, dm_context.min_version, is_common_handle); - auto other_stable_id = dm_context.storage_pool.newMetaPageId(); + auto other_stable_id = dm_context.page_id_generator.newMetaPageId(); other_stable = createNewStable(dm_context, schema_snap, other_data, other_stable_id, wbs); } @@ -1075,8 +1075,8 @@ SegmentPair Segment::applySplit(DMContext & dm_context, // // Created references to tail pages' pages in "log" storage, we need to write them down. wbs.writeLogAndData(); - auto other_segment_id = dm_context.storage_pool.newMetaPageId(); - auto other_delta_id = dm_context.storage_pool.newMetaPageId(); + auto other_segment_id = dm_context.page_id_generator.newMetaPageId(); + auto other_delta_id = dm_context.page_id_generator.newMetaPageId(); auto my_delta = std::make_shared(delta->getId(), my_persisted_files, my_in_memory_files); auto other_delta = std::make_shared(other_delta_id, other_persisted_files, other_in_memory_files); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index e86e42191d0..3f540b1dd61 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -71,9 +71,6 @@ StoragePool::StoragePool(const String & name, StoragePathPool & path_pool, const path_pool.getPSDiskDelegatorMulti("meta"), extractConfig(settings, StorageType::Meta), global_ctx.getFileProvider())) - , max_log_page_id(0) - , max_data_page_id(0) - , max_meta_page_id(0) , global_context(global_ctx) {} @@ -82,10 +79,6 @@ void StoragePool::restore() log_storage->restore(); data_storage->restore(); meta_storage->restore(); - - max_log_page_id = log_storage->getMaxId(); - max_data_page_id = data_storage->getMaxId(); - max_meta_page_id = meta_storage->getMaxId(); } void StoragePool::drop() @@ -95,36 +88,6 @@ void StoragePool::drop() log_storage->drop(); } -PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who) -{ - // In case that there is a DTFile created on disk but TiFlash crashes without persisting the ID. - // After TiFlash process restored, the ID will be inserted into the stable delegator, but we may - // get a duplicated ID from the `storage_pool.data`. (tics#2756) - PageId dtfile_id; - do - { - dtfile_id = ++max_data_page_id; - - auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false); - fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, { - static size_t fail_point_called = 0; - if (existed_path.empty() && fail_point_called % 10 == 0) - { - existed_path = ""; - } - fail_point_called++; - }); - if (likely(existed_path.empty())) - { - break; - } - // else there is a DTFile with that id, continue to acquire a new ID. - LOG_WARNING(&Poco::Logger::get(who), - fmt::format("The DTFile is already exists, continute to acquire another ID. [path={}] [id={}]", existed_path, dtfile_id)); - } while (true); - return dtfile_id; -} - bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) { { @@ -155,5 +118,43 @@ bool StoragePool::gc(const Settings & settings, const Seconds & try_gc_period) return done_anything; } +void PageIdGenerator::restore(const StoragePool & storage_pool) +{ + max_log_page_id = storage_pool.log_storage->getMaxId(); + max_data_page_id = storage_pool.data_storage->getMaxId(); + max_meta_page_id = storage_pool.meta_storage->getMaxId(); +} + +PageId PageIdGenerator::newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who) +{ + // In case that there is a DTFile created on disk but TiFlash crashes without persisting the ID. + // After TiFlash process restored, the ID will be inserted into the stable delegator, but we may + // get a duplicated ID from the `storage_pool.data`. (tics#2756) + PageId dtfile_id; + do + { + dtfile_id = ++max_data_page_id; + + auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false); + fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, { + static size_t fail_point_called = 0; + if (existed_path.empty() && fail_point_called % 10 == 0) + { + existed_path = ""; + } + fail_point_called++; + }); + if (likely(existed_path.empty())) + { + break; + } + // else there is a DTFile with that id, continue to acquire a new ID. + LOG_FMT_WARNING(&Poco::Logger::get(who), + "The DTFile is already exists, continute to acquire another ID. [path={}] [id={}]", + existed_path, + dtfile_id); + } while (true); + return dtfile_id; +} } // namespace DM } // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool.h index 97f04e3d18e..fc1e2ec076c 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool.h @@ -28,15 +28,6 @@ class StoragePool : private boost::noncopyable void restore(); - PageId maxLogPageId() { return max_log_page_id; } - PageId maxDataPageId() { return max_data_page_id; } - PageId maxMetaPageId() { return max_meta_page_id; } - - PageId newLogPageId() { return ++max_log_page_id; } - PageId newMetaPageId() { return ++max_meta_page_id; } - - PageId newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who); - PageStoragePtr log() { return log_storage; } PageStoragePtr data() { return data_storage; } PageStoragePtr meta() { return meta_storage; } @@ -51,15 +42,33 @@ class StoragePool : private boost::noncopyable PageStoragePtr data_storage; PageStoragePtr meta_storage; - std::atomic max_log_page_id; - std::atomic max_data_page_id; - std::atomic max_meta_page_id; - std::atomic last_try_gc_time = Clock::now(); std::mutex mutex; const Context & global_context; + + friend class PageIdGenerator; +}; + +class PageIdGenerator : private boost::noncopyable +{ +public: + PageIdGenerator() = default; + + void restore(const StoragePool & storage_pool); + + PageId newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who); + + PageId maxMetaPageId() { return max_meta_page_id; } + + PageId newLogPageId() { return ++max_log_page_id; } + PageId newMetaPageId() { return ++max_meta_page_id; } + +private: + std::atomic max_log_page_id = 0; + std::atomic max_data_page_id = 0; + std::atomic max_meta_page_id = 0; }; struct StorageSnapshot : private boost::noncopyable diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp index 8b63e9c1485..6721797e0da 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp @@ -66,7 +66,9 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); storage_pool = std::make_unique("test.t1", *storage_path_pool, *db_context, db_context->getSettingsRef()); + page_id_generator = std::make_unique(); storage_pool->restore(); + page_id_generator->restore(*storage_pool); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); @@ -81,6 +83,7 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic dm_context = std::make_unique(*db_context, *storage_path_pool, *storage_pool, + *page_id_generator, 0, /*min_version_*/ 0, settings.not_compress_columns, @@ -97,6 +100,7 @@ class DeltaValueSpaceTest : public DB::base::TiFlashStorageTestBasic /// all these var lives as ref in dm_context std::unique_ptr storage_path_pool; std::unique_ptr storage_pool; + std::unique_ptr page_id_generator; ColumnDefinesPtr table_columns; DM::DeltaMergeStore::Settings settings; /// dm_context @@ -127,7 +131,7 @@ void appendColumnFileBigToDeltaValueSpace(DMContext & context, ColumnDefines col { Block block = DMTestEnv::prepareSimpleWriteBlock(rows_start, rows_start + rows_num, false, tso); auto delegator = context.path_pool.getStableDiskDelegator(); - auto file_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + auto file_id = context.page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); auto input_stream = std::make_shared(block); auto store_path = delegator.choosePath(); auto dmfile diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index 4aef503fac6..125dfdaf4da 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -91,6 +91,7 @@ class DMFile_Test parent_path = TiFlashStorageTestBasic::getTemporaryPath(); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFile_Test", false)); storage_pool = std::make_unique("test.t1", *path_pool, *db_context, db_context->getSettingsRef()); + page_id_generator = std::make_unique(); dm_file = DMFile::create(1, parent_path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); @@ -109,6 +110,7 @@ class DMFile_Test *db_context, *path_pool, *storage_pool, + *page_id_generator, /*hash_salt*/ 0, 0, settings.not_compress_columns, @@ -136,6 +138,7 @@ class DMFile_Test /// all these var live as ref in dm_context std::unique_ptr path_pool; std::unique_ptr storage_pool; + std::unique_ptr page_id_generator; ColumnDefinesPtr table_columns_; DeltaMergeStore::Settings settings; @@ -1117,6 +1120,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); storage_pool = std::make_unique("test.t1", *path_pool, *db_context, DB::Settings()); + page_id_generator = std::make_unique(); dm_file = DMFile::create(0, path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); column_cache_ = std::make_shared(); @@ -1137,6 +1141,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic *db_context, *path_pool, *storage_pool, + *page_id_generator, /*hash_salt*/ 0, 0, settings.not_compress_columns, @@ -1156,6 +1161,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic /// all these var live as ref in dm_context std::unique_ptr path_pool; std::unique_ptr storage_pool; + std::unique_ptr page_id_generator; ColumnDefinesPtr table_columns_; DeltaMergeStore::Settings settings; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 9ba27f6d772..52c92712e4b 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -61,11 +61,13 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic TiFlashStorageTestBasic::reload(std::move(db_settings)); storage_path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t1", false)); storage_pool = std::make_unique("test.t1", *storage_path_pool, *db_context, db_context->getSettingsRef()); + page_id_generator = std::make_unique(); storage_pool->restore(); + page_id_generator->restore(*storage_pool); ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); - return Segment::newSegment(*dm_context_, table_columns_, RowKeyRange::newAll(false, 1), storage_pool->newMetaPageId(), 0); + return Segment::newSegment(*dm_context_, table_columns_, RowKeyRange::newAll(false, 1), page_id_generator->newMetaPageId(), 0); } // setColumns should update dm_context at the same time @@ -76,6 +78,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic dm_context_ = std::make_unique(*db_context, *storage_path_pool, *storage_pool, + *page_id_generator, 0, /*min_version_*/ 0, settings.not_compress_columns, @@ -92,6 +95,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic /// all these var lives as ref in dm_context std::unique_ptr storage_path_pool; std::unique_ptr storage_pool; + std::unique_ptr page_id_generator; ColumnDefinesPtr table_columns_; DM::DeltaMergeStore::Settings settings; /// dm_context @@ -1232,7 +1236,7 @@ class Segment_test_2 : public Segment_test std::pair> genDMFile(DMContext & context, const Block & block) { auto delegator = context.path_pool.getStableDiskDelegator(); - auto file_id = context.storage_pool.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); + auto file_id = context.page_id_generator.newDataPageIdForDTFile(delegator, __PRETTY_FUNCTION__); auto input_stream = std::make_shared(block); auto store_path = delegator.choosePath(); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index 4b494e0f548..3846faa19d1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -40,12 +40,14 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic TiFlashStorageTestBasic::reload(std::move(db_settings)); path_pool = std::make_unique(db_context->getPathPool().withTable("test", "t", false)); storage_pool = std::make_unique("test.t1", *path_pool, *db_context, db_context->getSettingsRef()); + page_id_generator = std::make_unique(); storage_pool->restore(); + page_id_generator->restore(*storage_pool); if (!cols) cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); setColumns(cols); - auto segment_id = storage_pool->newMetaPageId(); + auto segment_id = page_id_generator->newMetaPageId(); return Segment::newSegment(*dm_context_, table_columns_, RowKeyRange::newAll(is_common_handle, rowkey_column_size), segment_id, 0); } @@ -57,6 +59,7 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic dm_context_ = std::make_unique(*db_context, *path_pool, *storage_pool, + *page_id_generator, 0, /*min_version_*/ 0, settings.not_compress_columns, @@ -73,6 +76,7 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic /// all these var lives as ref in dm_context std::unique_ptr path_pool; std::unique_ptr storage_pool; + std::unique_ptr page_id_generator; ColumnDefinesPtr table_columns_; DM::DeltaMergeStore::Settings settings; /// dm_context