Feature:Online Query and Dynamic Modification of Table-level RocksDB …

…Options (#1488)

#1488
Complete the dynamic setting function of num_levels and write_buffer_size option.I use rocksdb.write_buffer_size as a dynamically modifiable parameter and rocksdb.num_levels as a non-dynamically modifiable parameter to test my idea.
1. Providing online modification functionality for Pegasus table-level RocksDB configuration:
   1. The common RocksDB options can still be configured through the `config.ini` file.
   2. For the RocksDB options that support dynamic modification, they can be modified using the `create` or `set_app_envs` command.
   3. For the RocksDB options that do not support dynamic modification, they can be modified using the `create` command.
2. Providing online query functionality for Pegasus table-level RocksDB configuration:
   1. Following the design principle of app environment (`app env`), only the RocksDB options set through the `create` or `set_app_envs` command will be available for online querying.
   2. The remaining RocksDB option information is located in the `[pegasus.server]` section of the `config.ini` file.
- Unit test
Unit test is add in meta_app_envs_test.update_app_envs_test
- Manual test (add detailed scripts or steps below)
Pegasus shell case:
create lpf -e rocksdb.num_levels=12,rocksdb.write_buffer_size=100
create lpf -e rocksdb.num_levels=5,rocksdb.write_buffer_size=100
create lpf -e rocksdb.num_levels=5,rocksdb.write_buffer_size=33554432
set_app_envs rocksdb.write_buffer_size  67108864
set_app_envs rocksdb.write_buffer_size  100
get_app_envs

src/common/replication_common.cpp

@@ -29,6 +29,7 @@
 #include <algorithm>
 #include <fstream>
 #include <memory>
+#include <set>
 
 #include "common/gpid.h"
 #include "common/replica_envs.h"

src/meta/app_env_validator.cpp

@@ -35,10 +35,15 @@ bool validate_app_envs(const std::map<std::string, std::string> &envs)
 {
     if (envs.size() == 0)
         return true;
-    // check app envs information
+    // check rocksdb app envs information
     std::string hint_message;
     bool all_envs_vaild = true;
     for (auto &it : envs) {
+        if (replica_envs::ROCKSDB_STATIC_OPTIONS.find(it.first) ==
+                replica_envs::ROCKSDB_STATIC_OPTIONS.end() &&
+            replica_envs::ROCKSDB_DYNAMIC_OPTIONS.find(it.first) ==
+                replica_envs::ROCKSDB_DYNAMIC_OPTIONS.end())
+            continue;
         if (!validate_app_env(it.first, it.second, hint_message)) {
             LOG_WARNING(
                 "app env {}={} is invaild, hint_message:{}", it.first, it.second, hint_message);

src/meta/server_state.cpp

@@ -2757,12 +2757,6 @@ void server_state::set_app_envs(const app_env_rpc &env_rpc)
             return;
         }
 
-        if (replica_envs::ROCKSDB_STATIC_OPTIONS.find(keys[i]) !=
-            replica_envs::ROCKSDB_STATIC_OPTIONS.end()) {
-            env_rpc.response().err = ERR_INVALID_PARAMETERS;
-            env_rpc.response().hint_message = "static rocksdb option only can set by create app";
-            return;
-        }
         os << keys[i] << "=" << values[i];
     }
     LOG_INFO("set app envs for app({}) from remote({}): kvs = {}",

src/meta/test/meta_app_envs_test.cpp

@@ -153,11 +153,6 @@ TEST_F(meta_app_envs_test, update_app_envs_test)
          "rocksdb.write_buffer_size suggest set val in range [33554432, 536870912]",
          "536870912"},
         {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "67108864", ERR_OK, "", "67108864"},
-        {replica_envs::ROCKSDB_NUM_LEVELS,
-         "5",
-         ERR_INVALID_PARAMETERS,
-         "static rocksdb option only can set by create app",
-         "5"},
         {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION,
          "200",
          ERR_OK,

src/server/pegasus_server_impl.cpp

@@ -42,6 +42,7 @@
 #include <list>
 #include <mutex>
 #include <ostream>
+#include <set>
 
 #include "base/pegasus_key_schema.h"
 #include "base/pegasus_utils.h"
@@ -2655,8 +2656,10 @@ void pegasus_server_impl::update_rocksdb_options_before_create_replica(
         auto find = envs.find(option);
         bool is_set = false;
         if (option.compare(ROCKSDB_NUM_LEVELS) == 0 && find != envs.end()) {
-            dsn::buf2int32(find->second, _data_cf_opts.num_levels);
+            int32_t val=0;
+            if(!dsn::buf2int32(find->second, val))continue;
             is_set = true;
+            _data_cf_opts.num_levels = val;
         }
 
         if (is_set)
@@ -2666,10 +2669,11 @@ void pegasus_server_impl::update_rocksdb_options_before_create_replica(
     for (auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) {
         auto find = envs.find(option);
         bool is_set = false;
-
         if (option.compare(ROCKSDB_WRITE_BUFFER_SIZE) == 0 && find != envs.end()) {
-            dsn::buf2uint64(find->second, _data_cf_opts.write_buffer_size);
+            uint64_t val = 0;
+            if(!dsn::buf2uint64(find->second, val))continue;
             is_set = true;
+            _data_cf_opts.write_buffer_size = static_cast<size_t>(val);
         }
         if (is_set)
             LOG_INFO("Reset {} \"{}\" succeed", find->first, find->second);