feat: add page_stress_testing as a subcommand of tiflash (#5038)

close #5037
pingcap · Jun 10, 2022 · a9b322a · a9b322a
1 parent ba725cc
commit a9b322a
Show file tree

Hide file tree

Showing 47 changed files with 270 additions and 212 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -432,9 +432,6 @@ else (ENABLE_FAILPOINTS)
     message (STATUS "Failpoints are disabled")
 endif (ENABLE_FAILPOINTS)
 
-# Enable PageStorage V3 test.
-option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON)
-
 # Flags for test coverage
 option (TEST_COVERAGE "Enables flags for test coverage" OFF)
 option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF)

diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
@@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
 add_headers_and_sources(dbms src/Storages/Page/V2/gc)
 add_headers_and_sources(dbms src/WindowFunctions)
 add_headers_and_sources(dbms src/TiDB/Schema)
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(dbms src/Storages/Page/V3)
-    add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
-    add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
-    add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
-endif()
+add_headers_and_sources(dbms src/Storages/Page/V3)
+add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
+add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
+add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
 add_headers_and_sources(dbms src/Storages/Page/)
 add_headers_and_sources(dbms src/TiDB)
 add_headers_and_sources(dbms src/Client)
@@ -323,6 +321,9 @@ if (ENABLE_TESTS)
     if (ENABLE_TIFLASH_DTWORKLOAD)
         target_link_libraries(bench_dbms dt-workload-lib)
     endif ()
+    if (ENABLE_TIFLASH_PAGEWORKLOAD)
+        target_link_libraries(bench_dbms page-workload-lib)
+    endif ()
 
     add_check(bench_dbms)
 endif ()

diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt
@@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL})
+option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL})
 
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
@@ -136,6 +137,9 @@ endif ()
 if (ENABLE_TIFLASH_DTWORKLOAD)
     target_link_libraries(tiflash dt-workload-lib)
 endif ()
+if (ENABLE_TIFLASH_PAGEWORKLOAD)
+    target_link_libraries(tiflash page-workload-lib)
+endif()
 if (ENABLE_TIFLASH_PAGECTL)
     target_link_libraries(tiflash page-ctl-lib)
 endif ()

diff --git a/dbms/src/Server/config_tools.h.in b/dbms/src/Server/config_tools.h.in
@@ -6,4 +6,5 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
 #cmakedefine01 ENABLE_TIFLASH_DTTOOL
 #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD
+#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD
 #cmakedefine01 ENABLE_TIFLASH_PAGECTL
diff --git a/dbms/src/Server/main.cpp b/dbms/src/Server/main.cpp
@@ -36,7 +36,10 @@
 #include <Server/DTTool/DTTool.h>
 #endif
 #if ENABLE_TIFLASH_DTWORKLOAD
-#include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
+#include <Storages/DeltaMerge/workload/DTWorkload.h>
+#endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+#include <Storages/Page/workload/PSWorkload.h>
 #endif
 #if ENABLE_TIFLASH_PAGECTL
 #include <Storages/Page/tools/PageCtl/PageStorageCtl.h>
@@ -107,6 +110,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] = {
 #if ENABLE_TIFLASH_DTWORKLOAD
     {"dtworkload", DB::DM::tests::DTWorkload::mainEntry},
 #endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+    {"pageworkload", DB::PS::tests::StressWorkload::mainEntry},
+#endif
 #if ENABLE_TIFLASH_PAGECTL
     {"pagectl", DB::PageStorageCtl::mainEntry},
 #endif

diff --git a/dbms/src/Storages/CMakeLists.txt b/dbms/src/Storages/CMakeLists.txt
@@ -15,16 +15,15 @@
 add_subdirectory (System)
 add_subdirectory (Page)
 add_subdirectory (DeltaMerge/File/dtpb)
-add_subdirectory (DeltaMerge/tools)
+add_subdirectory (DeltaMerge/workload)
+add_subdirectory (Page/workload)
 
 if (ENABLE_TESTS)
     add_subdirectory (tests EXCLUDE_FROM_ALL)
     add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL)
     add_subdirectory (Page/V2/tests EXCLUDE_FROM_ALL)
-    if (ENABLE_V3_PAGESTORAGE)
-        add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
-        add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
-    endif ()
+    add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
+    add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
     add_subdirectory (DeltaMerge/tests EXCLUDE_FROM_ALL)
 endif ()
 
diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp b/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp
diff --git a/.../DeltaMerge/tools/workload/CMakeLists.txt → ...orages/DeltaMerge/workload/CMakeLists.txt b/.../DeltaMerge/tools/workload/CMakeLists.txt → ...orages/DeltaMerge/workload/CMakeLists.txt
@@ -18,6 +18,3 @@ set(dt-workload-src MainEntry.cpp DTWorkload.cpp KeyGenerator.cpp TableGenerator
 
 add_library(dt-workload-lib ${dt-workload-src})
 target_link_libraries(dt-workload-lib dbms clickhouse_functions clickhouse-server-lib)
-
-add_executable(dt-workload Main.cpp ${dt-workload-src})
-target_link_libraries(dt-workload dbms gtest clickhouse_functions clickhouse-server-lib)
diff --git a/.../DeltaMerge/tools/workload/DTWorkload.cpp → ...orages/DeltaMerge/workload/DTWorkload.cpp b/.../DeltaMerge/tools/workload/DTWorkload.cpp → ...orages/DeltaMerge/workload/DTWorkload.cpp
@@ -19,16 +19,16 @@
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Storages/DeltaMerge/DeltaMergeStore.h>
 #include <Storages/DeltaMerge/Filter/RSOperator.h>
-#include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
-#include <Storages/DeltaMerge/tools/workload/DataGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/Handle.h>
-#include <Storages/DeltaMerge/tools/workload/KeyGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/Limiter.h>
-#include <Storages/DeltaMerge/tools/workload/Options.h>
-#include <Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/TableGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/TimestampGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/Utils.h>
+#include <Storages/DeltaMerge/workload/DTWorkload.h>
+#include <Storages/DeltaMerge/workload/DataGenerator.h>
+#include <Storages/DeltaMerge/workload/Handle.h>
+#include <Storages/DeltaMerge/workload/KeyGenerator.h>
+#include <Storages/DeltaMerge/workload/Limiter.h>
+#include <Storages/DeltaMerge/workload/Options.h>
+#include <Storages/DeltaMerge/workload/ReadColumnsGenerator.h>
+#include <Storages/DeltaMerge/workload/TableGenerator.h>
+#include <Storages/DeltaMerge/workload/TimestampGenerator.h>
+#include <Storages/DeltaMerge/workload/Utils.h>
 #include <TestUtils/TiFlashTestEnv.h>
 #include <cpptoml.h>
 

diff --git a/...es/DeltaMerge/tools/workload/DTWorkload.h → ...Storages/DeltaMerge/workload/DTWorkload.h b/...es/DeltaMerge/tools/workload/DTWorkload.h → ...Storages/DeltaMerge/workload/DTWorkload.h
@@ -73,7 +73,7 @@ class ThreadStat
 class Statistics
 {
 public:
-    Statistics(int write_thread_count = 0, int read_thread_count = 0)
+    explicit Statistics(int write_thread_count = 0, int read_thread_count = 0)
         : init_ms(0)
         , write_stats(write_thread_count)
         , read_stats(read_thread_count)

diff --git a/...ltaMerge/tools/workload/DataGenerator.cpp → ...ges/DeltaMerge/workload/DataGenerator.cpp b/...ltaMerge/tools/workload/DataGenerator.cpp → ...ges/DeltaMerge/workload/DataGenerator.cpp
@@ -13,11 +13,11 @@
 // limitations under the License.
 
 #include <DataTypes/DataTypeEnum.h>
-#include <Storages/DeltaMerge/tools/workload/DataGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/KeyGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/Options.h>
-#include <Storages/DeltaMerge/tools/workload/TableGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/TimestampGenerator.h>
+#include <Storages/DeltaMerge/workload/DataGenerator.h>
+#include <Storages/DeltaMerge/workload/KeyGenerator.h>
+#include <Storages/DeltaMerge/workload/Options.h>
+#include <Storages/DeltaMerge/workload/TableGenerator.h>
+#include <Storages/DeltaMerge/workload/TimestampGenerator.h>
 #include <fmt/ranges.h>
 
 #include <random>
@@ -33,7 +33,7 @@ class RandomDataGenerator : public DataGenerator
         , rand_gen(std::random_device()())
     {}
 
-    virtual std::tuple<Block, uint64_t> get(uint64_t key) override
+    std::tuple<Block, uint64_t> get(uint64_t key) override
     {
         Block block;
         // Generate 'rowkeys'.
@@ -227,7 +227,9 @@ class RandomDataGenerator : public DataGenerator
     struct tm randomLocalTime()
     {
         time_t t = randomUTCTimestamp();
-        struct tm res;
+        struct tm res
+        {
+        };
         if (localtime_r(&t, &res) == nullptr)
         {
             throw std::invalid_argument(fmt::format("localtime_r({}) ret {}", t, strerror(errno)));

diff --git a/...DeltaMerge/tools/workload/DataGenerator.h → ...rages/DeltaMerge/workload/DataGenerator.h b/...DeltaMerge/tools/workload/DataGenerator.h → ...rages/DeltaMerge/workload/DataGenerator.h
@@ -27,7 +27,7 @@ class DataGenerator
 public:
     static std::unique_ptr<DataGenerator> create(const WorkloadOptions & opts, const TableInfo & table_info, TimestampGenerator & ts_gen);
     virtual std::tuple<Block, uint64_t> get(uint64_t key) = 0;
-    virtual ~DataGenerator() {}
+    virtual ~DataGenerator() = default;
 };
 
 std::string blockToString(const Block & block);

diff --git a/...orages/DeltaMerge/tools/workload/Handle.h → ...src/Storages/DeltaMerge/workload/Handle.h b/...orages/DeltaMerge/tools/workload/Handle.h → ...src/Storages/DeltaMerge/workload/Handle.h
@@ -16,7 +16,7 @@
 
 #include <Encryption/PosixRandomAccessFile.h>
 #include <Encryption/PosixWritableFile.h>
-#include <Storages/DeltaMerge/tools/workload/TableGenerator.h>
+#include <Storages/DeltaMerge/workload/TableGenerator.h>
 #include <fcntl.h>
 
 #include <memory>
@@ -40,7 +40,7 @@ class HandleLock
     static constexpr uint64_t default_lock_count = 4096;
 
     static std::unique_ptr<HandleLock> create(const TableInfo & table_info);
-    HandleLock(uint64_t lock_count = default_lock_count)
+    explicit HandleLock(uint64_t lock_count = default_lock_count)
         : rmtxs(lock_count)
     {}
 
@@ -51,14 +51,14 @@ class HandleLock
 
     std::vector<std::unique_lock<std::recursive_mutex>> getLocks(const std::vector<uint64_t> & handles)
     {
-        std::vector<uint64_t> indexes;
+        std::vector<uint64_t> indexes(handles.size());
         for (const auto & h : handles)
         {
             indexes.push_back(index(h));
         }
         // Sort mutex indexes to avoid dead lock.
         sort(indexes.begin(), indexes.end());
-        std::vector<std::unique_lock<std::recursive_mutex>> locks;
+        std::vector<std::unique_lock<std::recursive_mutex>> locks(indexes.size());
         for (auto i : indexes)
         {
             locks.push_back(getLockByIndex(i));
@@ -105,7 +105,7 @@ class HandleTable
         std::lock_guard lock(mtx);
         handle_to_ts[handle] = ts;
         Record r{handle, ts};
-        if (wal != nullptr && wal->write((char *)&r, sizeof(r)) != sizeof(r))
+        if (wal != nullptr && wal->write(reinterpret_cast<char *>(&r), sizeof(r)) != sizeof(r))
         {
             throw std::runtime_error(fmt::format("write ret {}", strerror(errno)));
         }
@@ -134,8 +134,8 @@ class HandleTable
         try
         {
             PosixRandomAccessFile f(fname, -1);
-            Record r;
-            while (f.read((char *)&r, sizeof(r)) == sizeof(r))
+            Record r{};
+            while (f.read(reinterpret_cast<char *>(&r), sizeof(r)) == sizeof(r))
             {
                 handle_to_ts[r.handle] = r.ts;
             }
@@ -156,7 +156,7 @@ class HandleTable
         for (const auto & pa : handle_to_ts)
         {
             Record r{pa.first, pa.second};
-            if (f.write((char *)&r, sizeof(r)) != sizeof(r))
+            if (f.write(reinterpret_cast<char *>(&r), sizeof(r)) != sizeof(r))
             {
                 throw std::runtime_error(fmt::format("write ret {}", strerror(errno)));
             }
@@ -191,7 +191,7 @@ class SharedHandleTable
 public:
     static constexpr uint64_t default_shared_count = 4096;
 
-    SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count)
+    explicit SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count)
         : tables(shared_cnt)
     {
         uint64_t max_key_count_per_shared = max_key_count / default_shared_count + 1;

diff --git a/...eltaMerge/tools/workload/KeyGenerator.cpp → ...ages/DeltaMerge/workload/KeyGenerator.cpp b/...eltaMerge/tools/workload/KeyGenerator.cpp → ...ages/DeltaMerge/workload/KeyGenerator.cpp
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <Storages/DeltaMerge/tools/workload/KeyGenerator.h>
-#include <Storages/DeltaMerge/tools/workload/Options.h>
+#include <Storages/DeltaMerge/workload/KeyGenerator.h>
+#include <Storages/DeltaMerge/workload/Options.h>
 #include <fmt/core.h>
 
 #include <atomic>
@@ -31,7 +31,7 @@ class IncrementalKeyGenerator : public KeyGenerator
         , key(0)
     {}
 
-    virtual uint64_t get64() override
+    uint64_t get64() override
     {
         return key.fetch_add(1, std::memory_order_relaxed) % key_count + start_key;
     }
@@ -54,7 +54,7 @@ class UniformDistributionKeyGenerator : public KeyGenerator
         , uniform_dist(0, key_count)
     {}
 
-    virtual uint64_t get64() override
+    uint64_t get64() override
     {
         std::lock_guard lock(mtx);
         return uniform_dist(rand_gen);
@@ -78,7 +78,7 @@ class NormalDistributionKeyGenerator : public KeyGenerator
         , normal_dist(key_count / 2.0, key_count / 20.0)
     {}
 
-    virtual uint64_t get64() override
+    uint64_t get64() override
     {
         std::lock_guard lock(mtx);
         return normal_dist(rand_gen);

diff --git a/.../DeltaMerge/tools/workload/KeyGenerator.h → ...orages/DeltaMerge/workload/KeyGenerator.h b/.../DeltaMerge/tools/workload/KeyGenerator.h → ...orages/DeltaMerge/workload/KeyGenerator.h
@@ -23,8 +23,8 @@ class KeyGenerator
 public:
     static std::unique_ptr<KeyGenerator> create(const WorkloadOptions & opts);
 
-    KeyGenerator() {}
-    virtual ~KeyGenerator() {}
+    KeyGenerator() = default;
+    virtual ~KeyGenerator() = default;
 
     virtual uint64_t get64() = 0;
 };

diff --git a/...ges/DeltaMerge/tools/workload/Limiter.cpp → .../Storages/DeltaMerge/workload/Limiter.cpp b/...ges/DeltaMerge/tools/workload/Limiter.cpp → .../Storages/DeltaMerge/workload/Limiter.cpp
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 #include <Encryption/RateLimiter.h>
-#include <Storages/DeltaMerge/tools/workload/Limiter.h>
-#include <Storages/DeltaMerge/tools/workload/Options.h>
+#include <Storages/DeltaMerge/workload/Limiter.h>
+#include <Storages/DeltaMerge/workload/Options.h>
 #include <fmt/core.h>
 
 #include <cmath>
@@ -24,10 +24,10 @@ namespace DB::DM::tests
 class ConstantLimiter : public Limiter
 {
 public:
-    ConstantLimiter(uint64_t rate_per_sec)
+    explicit ConstantLimiter(uint64_t rate_per_sec)
         : limiter(rate_per_sec, LimiterType::UNKNOW)
     {}
-    virtual void request() override
+    void request() override
     {
         limiter.request(1);
     }
@@ -38,7 +38,7 @@ class ConstantLimiter : public Limiter
 
 std::unique_ptr<Limiter> Limiter::create(const WorkloadOptions & opts)
 {
-    uint64_t per_sec = std::ceil(static_cast<double>(opts.max_write_per_sec / opts.write_thread_count));
+    uint64_t per_sec = std::ceil(opts.max_write_per_sec * 1.0 / opts.write_thread_count);
     return std::make_unique<ConstantLimiter>(per_sec);
 }
 

diff --git a/...rages/DeltaMerge/tools/workload/Limiter.h → ...rc/Storages/DeltaMerge/workload/Limiter.h b/...rages/DeltaMerge/tools/workload/Limiter.h → ...rc/Storages/DeltaMerge/workload/Limiter.h
@@ -23,6 +23,6 @@ class Limiter
 public:
     static std::unique_ptr<Limiter> create(const WorkloadOptions & opts);
     virtual void request() = 0;
-    virtual ~Limiter() {}
+    virtual ~Limiter() = default;
 };
 } // namespace DB::DM::tests
diff --git a/...s/DeltaMerge/tools/workload/MainEntry.cpp → ...torages/DeltaMerge/workload/MainEntry.cpp b/...s/DeltaMerge/tools/workload/MainEntry.cpp → ...torages/DeltaMerge/workload/MainEntry.cpp
@@ -14,10 +14,10 @@
 
 #include <Common/Exception.h>
 #include <Common/Logger.h>
-#include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
-#include <Storages/DeltaMerge/tools/workload/Handle.h>
-#include <Storages/DeltaMerge/tools/workload/Options.h>
-#include <Storages/DeltaMerge/tools/workload/Utils.h>
+#include <Storages/DeltaMerge/workload/DTWorkload.h>
+#include <Storages/DeltaMerge/workload/Handle.h>
+#include <Storages/DeltaMerge/workload/Options.h>
+#include <Storages/DeltaMerge/workload/Utils.h>
 #include <Storages/PathPool.h>
 #include <TestUtils/TiFlashTestEnv.h>
 #include <common/logger_useful.h>