Merge remote-tracking branch 'pingcap/master' into pch

pingcap · Jun 7, 2022 · 665f621 · 665f621
2 parents dae201b + e3a4412
commit 665f621
Show file tree

Hide file tree

Showing 25 changed files with 306 additions and 155 deletions.
diff --git a/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp b/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp
@@ -228,7 +228,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery)
 
             if (done_with_set && done_with_join && done_with_table)
             {
-                if (IProfilingBlockInputStream * profiling_in = dynamic_cast<IProfilingBlockInputStream *>(&*subquery.source))
+                if (auto * profiling_in = dynamic_cast<IProfilingBlockInputStream *>(&*subquery.source))
                     profiling_in->cancel(false);
 
                 break;
@@ -248,7 +248,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery)
         watch.stop();
 
         size_t head_rows = 0;
-        if (IProfilingBlockInputStream * profiling_in = dynamic_cast<IProfilingBlockInputStream *>(&*subquery.source))
+        if (auto * profiling_in = dynamic_cast<IProfilingBlockInputStream *>(&*subquery.source))
         {
             const BlockStreamProfileInfo & profile_info = profiling_in->getProfileInfo();
 
@@ -272,7 +272,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery)
                 if (subquery.table)
                     msg.fmtAppend("Table with {} rows. ", head_rows);
 
-                msg.fmtAppend("In {.3f} sec. ", watch.elapsedSeconds());
+                msg.fmtAppend("In {:.3f} sec. ", watch.elapsedSeconds());
                 msg.fmtAppend("using {} threads.", subquery.join ? subquery.join->getBuildConcurrency() : 1);
                 return msg.toString();
             };

diff --git a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp
@@ -17,8 +17,8 @@
 #include <Poco/Logger.h>
 #include <Storages/ColumnsDescription.h>
 #include <Storages/DeltaMerge/RowKeyRange.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/DeltaMerge/tests/MultiSegmentTestUtil.h>
-#include <Storages/DeltaMerge/tests/dm_basic_include.h>
 #include <Storages/StorageDeltaMerge.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/tests/TiFlashStorageTestBasic.h>

diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.cpp b/dbms/src/Interpreters/IDAsPathUpgrader.cpp
@@ -487,6 +487,7 @@ bool IDAsPathUpgrader::needUpgrade()
         if (db_info.engine != "TiFlash")
         {
             has_old_db_engine = true;
+            LOG_FMT_INFO(log, "Find old style of database engine, doing upgrade [path={}] [engine={}]", database_metadata_file, db_info.engine);
         }
     }
 

diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h
@@ -272,7 +272,7 @@ struct Settings
     M(SettingUInt64, dt_segment_delta_small_column_file_size, 8388608, "Determine whether a column file in delta is small or not. 8MB by default.")                                                                                     \
     M(SettingUInt64, dt_segment_stable_pack_rows, DEFAULT_MERGE_BLOCK_SIZE, "Expected stable pack rows in DeltaTree Engine.")                                                                                                           \
     M(SettingFloat, dt_segment_wait_duration_factor, 1, "The factor of wait duration in a write stall.")                                                                                                                                \
-    M(SettingUInt64, dt_bg_gc_check_interval, 5, "Background gc thread check interval, the unit is second.")                                                                                                                            \
+    M(SettingUInt64, dt_bg_gc_check_interval, 60, "Background gc thread check interval, the unit is second.")                                                                                                                            \
     M(SettingInt64, dt_bg_gc_max_segments_to_check_every_round, 100, "Max segments to check in every gc round, value less than or equal to 0 means gc no segments.")                                                                    \
     M(SettingFloat, dt_bg_gc_ratio_threhold_to_trigger_gc, 1.2, "Trigger segment's gc when the ratio of invalid version exceed this threhold. Values smaller than or equal to 1.0 means gc all "                                        \
                                                                 "segments")                                                                                                                                                             \

diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp
@@ -515,21 +515,9 @@ void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks
     }
     case PageStorageRunMode::MIX_MODE:
     {
-        // When PageStorage run as Mix Mode.
-        // We need both get alive pages from V2 and V3 which will feedback for the DM.
-        // But V2 and V3 won't GC in the same time. So V3 need proxy V2 external pages callback.
-        // When V3 GC happend, scan the external pages from V3, in remover will scanner all of external pages from V2.
-        ExternalPageCallbacks mix_mode_callbacks;
-
-        mix_mode_callbacks.scanner = callbacks.scanner;
-        mix_mode_callbacks.remover = [this, callbacks](const ExternalPageCallbacks::PathAndIdsVec & path_and_ids_vec, const std::set<PageId> & valid_ids) {
-            // ns_id won't used on V2
-            auto v2_valid_page_ids = data_storage_v2->getAliveExternalPageIds(ns_id);
-            v2_valid_page_ids.insert(valid_ids.begin(), valid_ids.end());
-            callbacks.remover(path_and_ids_vec, v2_valid_page_ids);
-        };
-        mix_mode_callbacks.ns_id = ns_id;
-        data_storage_v3->registerExternalPagesCallbacks(mix_mode_callbacks);
+        // We have transformed all pages from V2 to V3 in `restore`, so
+        // only need to register callbacks for V3.
+        data_storage_v3->registerExternalPagesCallbacks(callbacks);
         break;
     }
     default:

diff --git a/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt
@@ -21,7 +21,7 @@ macro(grep_gtest_sources BASE_DIR DST_VAR)
 endmacro()
 # attach all dm gtest sources
 grep_gtest_sources(${TiFlash_SOURCE_DIR}/dbms/src/Storages/DeltaMerge/tests dm_gtest_sources)
-add_executable(gtests_dm EXCLUDE_FROM_ALL ${dm_gtest_sources} dm_basic_include.h)
+add_executable(gtests_dm EXCLUDE_FROM_ALL ${dm_gtest_sources} DMTestEnv.h)
 target_link_libraries(gtests_dm gtest_main dbms clickhouse_functions)
 add_check(gtests_dm)
 

diff --git a/...rages/DeltaMerge/tests/dm_basic_include.h → ...src/Storages/DeltaMerge/tests/DMTestEnv.h b/...rages/DeltaMerge/tests/dm_basic_include.h → ...src/Storages/DeltaMerge/tests/DMTestEnv.h
diff --git a/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h b/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h
@@ -22,6 +22,7 @@
 #include <Storages/DeltaMerge/Range.h>
 #include <Storages/DeltaMerge/RowKeyRange.h>
 #include <Storages/DeltaMerge/Segment.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/StorageDeltaMerge.h>
 #include <Storages/tests/TiFlashStorageTestBasic.h>
 #include <TestUtils/TiFlashTestBasic.h>
@@ -30,7 +31,6 @@
 #include <cstdint>
 #include <vector>
 
-#include "dm_basic_include.h"
 
 namespace DB
 {
@@ -43,7 +43,6 @@ namespace DM
 {
 namespace tests
 {
-
 /// Helper class to test with multiple segments.
 /// You can call `prepareSegments` to prepare multiple segments. After that,
 /// you can use `verifyExpectedRowsForAllSegments` to verify the expectation for each segment.
@@ -157,4 +156,4 @@ class MultiSegmentTestUtil : private boost::noncopyable
 
 } // namespace tests
 } // namespace DM
-} // namespace DB
+} // namespace DB
diff --git a/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h b/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h
@@ -18,9 +18,9 @@
 #include <DataTypes/DataTypeString.h>
 #include <Poco/File.h>
 #include <Storages/DeltaMerge/DeltaMergeStore.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/DeltaMerge/tests/bank/IDGenerator.h>
 #include <Storages/DeltaMerge/tests/bank/SimpleDB.h>
-#include <Storages/DeltaMerge/tests/dm_basic_include.h>
 #include <TestUtils/TiFlashTestBasic.h>
 
 #include <cstddef>

diff --git a/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp b/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp
@@ -17,8 +17,8 @@
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
 #include <Storages/DeltaMerge/DeltaMergeStore.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h>
-#include <Storages/DeltaMerge/tests/dm_basic_include.h>
 #include <TestUtils/TiFlashTestBasic.h>
 
 #include <iostream>

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp
@@ -18,10 +18,9 @@
 #include <Storages/DeltaMerge/DeltaMergeDefines.h>
 #include <Storages/DeltaMerge/SchemaUpdate.h>
 #include <Storages/DeltaMerge/convertColumnTypeHelpers.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/Transaction/TiDB.h>
 
-#include "dm_basic_include.h"
-
 namespace DB
 {
 namespace DM

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp
@@ -15,7 +15,7 @@
 #include <DataStreams/BlocksListBlockInputStream.h>
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <Storages/DeltaMerge/PKSquashingBlockInputStream.h>
-#include <Storages/DeltaMerge/tests/dm_basic_include.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 
 namespace DB
 {

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
@@ -27,6 +27,8 @@
 #include <Storages/DeltaMerge/Filter/RSOperator.h>
 #include <Storages/DeltaMerge/PKSquashingBlockInputStream.h>
 #include <Storages/DeltaMerge/Segment.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
+#include <Storages/DeltaMerge/tests/MultiSegmentTestUtil.h>
 #include <Storages/tests/TiFlashStorageTestBasic.h>
 #include <TestUtils/FunctionTestUtils.h>
 #include <TestUtils/TiFlashTestBasic.h>
@@ -36,9 +38,6 @@
 #include <memory>
 #include <vector>
 
-#include "MultiSegmentTestUtil.h"
-#include "dm_basic_include.h"
-
 namespace DB
 {
 namespace FailPoints

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp
@@ -18,14 +18,13 @@
 #include <Storages/DeltaMerge/DeltaMergeStore.h>
 #include <Storages/DeltaMerge/File/DMFileBlockOutputStream.h>
 #include <Storages/DeltaMerge/Segment.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/tests/TiFlashStorageTestBasic.h>
 #include <TestUtils/TiFlashTestBasic.h>
 
 #include <memory>
 
-#include "dm_basic_include.h"
-
 namespace CurrentMetrics
 {
 extern const Metric DT_SnapshotOfRead;

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp
@@ -20,13 +20,12 @@
 #include <Storages/DeltaMerge/File/DMFileBlockOutputStream.h>
 #include <Storages/DeltaMerge/File/DMFileWriter.h>
 #include <Storages/DeltaMerge/RowKeyRange.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <Storages/tests/TiFlashStorageTestBasic.h>
 #include <TestUtils/FunctionTestUtils.h>
 
 #include <vector>
 
-#include "dm_basic_include.h"
-
 namespace DB
 {
 namespace FailPoints

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp
@@ -22,7 +22,7 @@
 #include <Storages/DeltaMerge/Index/RoughCheck.h>
 #include <Storages/DeltaMerge/Index/ValueComparison.h>
 #include <Storages/DeltaMerge/Segment.h>
-#include <Storages/DeltaMerge/tests/dm_basic_include.h>
+#include <Storages/DeltaMerge/tests/DMTestEnv.h>
 #include <TestUtils/TiFlashTestBasic.h>
 
 #include <ctime>
@@ -41,7 +41,7 @@ static const String DEFAULT_COL_NAME = "2020-09-26";
 class DMMinMaxIndexTest : public ::testing::Test
 {
 public:
-    DMMinMaxIndexTest() {}
+    DMMinMaxIndexTest() = default;
 
 protected:
     static void SetUpTestCase() {}