From 4b838e2d9ee338dc0d3ca86f69d76cb08f4529f2 Mon Sep 17 00:00:00 2001 From: Pengfan Lu Date: Thu, 29 Jun 2023 10:30:37 +0000 Subject: [PATCH] Thank you empiredan for the very detailed suggestion, I modified my code. Mainly the following aspects: 1. Improve the unit test to ensure that the new option can also be added correctly. 2. Add getter and setter methods for option. --- src/base/pegasus_const.cpp | 37 ++++ src/base/pegasus_const.h | 13 ++ src/meta/app_env_validator.cpp | 4 +- src/meta/test/meta_app_envs_test.cpp | 17 +- src/meta/test/meta_app_operation_test.cpp | 169 ++++++++++++++----- src/server/pegasus_server_impl.cpp | 65 +++---- src/server/test/pegasus_server_impl_test.cpp | 59 +++++++ 7 files changed, 289 insertions(+), 75 deletions(-) diff --git a/src/base/pegasus_const.cpp b/src/base/pegasus_const.cpp index 9fc4581ec6..857ce7d824 100644 --- a/src/base/pegasus_const.cpp +++ b/src/base/pegasus_const.cpp @@ -19,6 +19,13 @@ #include "pegasus_const.h" +#include +#include +#include +#include + +#include "utils/string_conv.h" + namespace pegasus { // should be same with items in dsn::backup_restore_constant @@ -112,4 +119,34 @@ const std::set ROCKSDB_DYNAMIC_OPTIONS = { const std::set ROCKSDB_STATIC_OPTIONS = { ROCKSDB_NUM_LEVELS, }; + +const std::unordered_map cf_opts_setters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + uint64_t val = 0; + if (!dsn::buf2uint64(str, val)) + return false; + option.write_buffer_size = static_cast(val); + return true; + }}, + {ROCKSDB_NUM_LEVELS, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + int32_t val = 0; + if (!dsn::buf2int32(str, val)) + return false; + option.num_levels = val; + return true; + }}, +}; + +const std::unordered_map cf_opts_getters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](/*out*/ std::string &str, const rocksdb::ColumnFamilyOptions &option) { + str = fmt::format("{}", option.write_buffer_size); + }}, + {ROCKSDB_NUM_LEVELS, + [](/*out*/ std::string &str, const rocksdb::ColumnFamilyOptions &option) { + str = fmt::format("{}", option.num_levels); + }}, +}; } // namespace pegasus diff --git a/src/base/pegasus_const.h b/src/base/pegasus_const.h index 7385c79411..c4db9662e5 100644 --- a/src/base/pegasus_const.h +++ b/src/base/pegasus_const.h @@ -19,8 +19,14 @@ #pragma once +#include #include #include +#include + +namespace rocksdb { +struct ColumnFamilyOptions; +} // namespace rocksdb namespace pegasus { @@ -81,4 +87,11 @@ extern const std::string ROCKSDB_NUM_LEVELS; extern const std::set ROCKSDB_DYNAMIC_OPTIONS; extern const std::set ROCKSDB_STATIC_OPTIONS; + +using cf_opts_setter = std::function; +extern const std::unordered_map cf_opts_setters; + +using cf_opts_getter = + std::function; +extern const std::unordered_map cf_opts_getters; } // namespace pegasus diff --git a/src/meta/app_env_validator.cpp b/src/meta/app_env_validator.cpp index fbc8eab467..7f197f5a0f 100644 --- a/src/meta/app_env_validator.cpp +++ b/src/meta/app_env_validator.cpp @@ -180,9 +180,9 @@ bool check_rocksdb_write_buffer_size(const std::string &env_value, std::string & hint_message = fmt::format("rocksdb.write_buffer_size cannot set this val: {}", env_value); return false; } - if (val < (32 << 20) || val > (512 << 20)) { + if (val < (16 << 20) || val > (512 << 20)) { hint_message = - fmt::format("rocksdb.write_buffer_size suggest set val in range [33554432, 536870912]"); + fmt::format("rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]"); return false; } return true; diff --git a/src/meta/test/meta_app_envs_test.cpp b/src/meta/test/meta_app_envs_test.cpp index c115f80770..93ffc6d7b3 100644 --- a/src/meta/test/meta_app_envs_test.cpp +++ b/src/meta/test/meta_app_envs_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "common/replica_envs.h" @@ -145,12 +146,12 @@ TEST_F(meta_app_envs_test, update_app_envs_test) {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "100", ERR_INVALID_PARAMETERS, - "rocksdb.write_buffer_size suggest set val in range [33554432, 536870912]", + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", "67108864"}, {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "636870912", ERR_INVALID_PARAMETERS, - "rocksdb.write_buffer_size suggest set val in range [33554432, 536870912]", + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", "536870912"}, {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "67108864", ERR_OK, "", "67108864"}, {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION, @@ -203,6 +204,18 @@ TEST_F(meta_app_envs_test, update_app_envs_test) ASSERT_EQ(app->envs.at(test.env_key), test.expect_value); } } + + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS are tested. + // Hint: Mainly verify the update_rocksdb_dynamic_options function. + std::map all_test_envs; + for (auto test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } } } // namespace replication diff --git a/src/meta/test/meta_app_operation_test.cpp b/src/meta/test/meta_app_operation_test.cpp index c3ddafa72c..92686010ce 100644 --- a/src/meta/test/meta_app_operation_test.cpp +++ b/src/meta/test/meta_app_operation_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,8 @@ class meta_app_operation_test : public meta_test_base error_code create_app_test(int32_t partition_count, int32_t replica_count, bool success_if_exist, - const std::string &app_name) + const std::string &app_name, + const std::map &envs = {}) { configuration_create_app_request create_request; configuration_create_app_response create_response; @@ -78,6 +80,7 @@ class meta_app_operation_test : public meta_test_base create_request.options.replica_count = replica_count; create_request.options.success_if_exist = success_if_exist; create_request.options.is_stateful = true; + create_request.options.envs = envs; auto result = fake_create_app(_ss.get(), create_request); fake_wait_rpc(result, create_response); @@ -362,6 +365,12 @@ TEST_F(meta_app_operation_test, create_app) // - wrong app_status dropping // - create succeed with app_status dropped // - create succeed with success_if_exist=true + // - wrong rocksdb.num_levels (< 1) + // - wrong rocksdb.num_levels (> 10) + // - create app with rocksdb.num_levels (= 5) succeed + // - wrong rocksdb.write_buffer_size (< (16<<20)) + // - wrong rocksdb.write_buffer_size (> (512<<20)) + // - create app with rocksdb.write_buffer_size (= (32<<20)) succeed struct create_test { std::string app_name; @@ -373,43 +382,106 @@ TEST_F(meta_app_operation_test, create_app) bool success_if_exist; app_status::type before_status; error_code expected_err; - } tests[] = {{APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}}; + std::map envs = {}; + } tests[] = { + {APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "0"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "11"}}}, + {APP_NAME + "_11", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.num_levels", "5"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1000"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1073741824"}}}, + {APP_NAME + "_12", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.write_buffer_size", "33554432"}}}, + }; clear_nodes(); @@ -453,13 +525,30 @@ TEST_F(meta_app_operation_test, create_app) } else if (test.before_status != app_status::AS_INVALID) { update_app_status(test.before_status); } - auto err = create_app_test( - test.partition_count, test.replica_count, test.success_if_exist, test.app_name); + auto err = create_app_test(test.partition_count, + test.replica_count, + test.success_if_exist, + test.app_name, + test.envs); ASSERT_EQ(err, test.expected_err); _ms->set_node_state(nodes, true); } + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS are + // tested. Hint: Mainly verify the validate_app_envs function. + std::map all_test_envs; + for (auto test : tests) { + all_test_envs.insert(test.envs.begin(), test.envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } // set FLAGS_min_allowed_replica_count successfully res = update_flag("min_allowed_replica_count", "2"); ASSERT_TRUE(res.is_ok()); diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 81c109f10e..7f6d7fc6c5 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -38,6 +38,7 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include @@ -2629,17 +2630,9 @@ pegasus_server_impl::get_restore_dir_from_env(const std::map &envs) { - if (envs.size() == 0) { + if (envs.empty()) { return; } - auto extract_option = [](const std::string &option) -> std::string { - std::stringstream ss(option); - std::string prefix, rocksdb_opt; - std::getline(ss, prefix, '.'); - std::getline(ss, rocksdb_opt); - LOG_INFO("Extract rocksdb dynamic opt ({}) from ({})", rocksdb_opt, option); - return rocksdb_opt; - }; std::unordered_map new_options; for (const auto &option : ROCKSDB_DYNAMIC_OPTIONS) { @@ -2647,11 +2640,15 @@ void pegasus_server_impl::update_rocksdb_dynamic_options( if (find == envs.end()) { continue; } - new_options[extract_option(option)] = find->second; + + std::vector args; + // split_args example: Parse "write_buffer_size" from "rocksdb.write_buffer_size" + dsn::utils::split_args(option.c_str(), args, '.'); + new_options[args[1]] = find->second; } // doing set option - if (new_options.empty() && set_options(new_options)) { + if (!new_options.empty() && set_options(new_options)) { LOG_INFO("Set rocksdb dynamic options success"); } } @@ -2659,7 +2656,7 @@ void pegasus_server_impl::update_rocksdb_dynamic_options( void pegasus_server_impl::set_rocksdb_options_before_creating( const std::map &envs) { - if (envs.size() == 0) { + if (envs.empty()) { return; } @@ -2668,17 +2665,15 @@ void pegasus_server_impl::set_rocksdb_options_before_creating( if (find == envs.end()) { continue; } - bool is_set = false; - if (option.compare(ROCKSDB_NUM_LEVELS) == 0) { - int32_t val = 0; - if (!dsn::buf2int32(find->second, val)) - continue; - is_set = true; - _data_cf_opts.num_levels = val; - } - if (is_set) - LOG_INFO("Set {} \"{}\" succeed", find->first, find->second); + auto setter = cf_opts_setters.find(option); + if (setter == cf_opts_setters.end()) { + LOG_WARNING("cannot find {} setter function, and set this option fail.", option); + continue; + } + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); + } } for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { @@ -2686,16 +2681,15 @@ void pegasus_server_impl::set_rocksdb_options_before_creating( if (find == envs.end()) { continue; } - bool is_set = false; - if (option.compare(ROCKSDB_WRITE_BUFFER_SIZE) == 0) { - uint64_t val = 0; - if (!dsn::buf2uint64(find->second, val)) - continue; - is_set = true; - _data_cf_opts.write_buffer_size = static_cast(val); + + auto setter = cf_opts_setters.find(option); + if (setter == cf_opts_setters.end()) { + LOG_WARNING("cannot find {} setter function, and set this option fail.", option); + continue; + } + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); } - if (is_set) - LOG_INFO("Set {} \"{}\" succeed", find->first, find->second); } } @@ -2731,6 +2725,14 @@ void pegasus_server_impl::update_app_envs_before_open_db( void pegasus_server_impl::query_app_envs(/*out*/ std::map &envs) { envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = _usage_scenario; + // write_buffer_size involves random values (refer to pegasus_server_impl::set_usage_scenario), + // so it can only be taken from _data_cf_opts + envs[ROCKSDB_WRITE_BUFFER_SIZE] = fmt::format("{}", _data_cf_opts.write_buffer_size); + + // Get Data ColumnFamilyOptions directly from _data_cf + rocksdb::ColumnFamilyDescriptor desc; + rocksdb::Status s = _data_cf->GetDescriptor(&desc); + envs[ROCKSDB_NUM_LEVELS] = fmt::format("{}", desc.options.num_levels); } void pegasus_server_impl::update_usage_scenario(const std::map &envs) @@ -3117,6 +3119,7 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario) void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts) { + LOG_INFO_PREFIX("Reset rocksdb envs options"); // Reset rocksdb option includes two aspects: // 1. Set usage_scenario related rocksdb options // 2. Rocksdb option set in app envs, consists of ROCKSDB_DYNAMIC_OPTIONS and diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 1363d8054b..ad28a822db 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -29,7 +29,9 @@ #include #include #include +#include #include +#include #include "pegasus_const.h" #include "pegasus_server_test_base.h" @@ -95,6 +97,51 @@ class pegasus_server_impl_test : public pegasus_server_test_base ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count); } } + + void test_open_db_with_rocksdb_envs(bool is_restart) + { + struct create_test + { + std::string env_key; + std::string env_value; + std::string expect_value; + } tests[] = { + {"rocksdb.num_levels", "5", "5"}, {"rocksdb.write_buffer_size", "33554432", "33554432"}, + }; + + std::map all_test_envs; + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS + // are tested. + for (auto test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } + + start(all_test_envs); + if (is_restart) { + _server->stop(false); + start(); + } + + std::map query_envs; + _server->query_app_envs(query_envs); + for (auto test : tests) { + auto iter = query_envs.find(test.env_key); + if (iter != query_envs.end()) { + ASSERT_EQ(iter->second, test.expect_value); + } else { + ASSERT_EQ(test.env_key, + fmt::format("query_app_envs not supported {}", test.env_key)); + } + } + } }; TEST_F(pegasus_server_impl_test, test_table_level_slow_query) @@ -137,6 +184,18 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } +TEST_F(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) +{ + // Hint: Verify the set_rocksdb_options_before_creating function by boolean is_restart=false. + test_open_db_with_rocksdb_envs(false); +} + +TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) +{ + // Hint: Verify the reset_rocksdb_options function by boolean is_restart=true. + test_open_db_with_rocksdb_envs(true); +} + TEST_F(pegasus_server_impl_test, test_stop_db_twice) { start();