diff --git a/rdsn b/rdsn index 7e3b0421a0a..b904de6f530 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 7e3b0421a0a40b92102357ed46581c8fbc2b4db2 +Subproject commit b904de6f530eaebae32b910ed394685af0093829 diff --git a/src/include/pegasus/version.h b/src/include/pegasus/version.h index 781dd269325..f9a1d6373a2 100644 --- a/src/include/pegasus/version.h +++ b/src/include/pegasus/version.h @@ -3,4 +3,4 @@ // can be found in the LICENSE file in the root directory of this source tree. #pragma once -#define PEGASUS_VERSION "1.12.SNAPSHOT" +#define PEGASUS_VERSION "1.13.SNAPSHOT" diff --git a/src/server/config.ini b/src/server/config.ini index e6946c5634c..9fc81804017 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -239,7 +239,8 @@ ;; WARNING: memory release may incur major performance downgrade when inproperly configured. ;; ensure this feature is only enabled when it's necessary. mem_release_enabled = false - mem_release_interval_ms = 86400000 + mem_release_check_interval_ms = 3600000 + mem_release_max_reserved_mem_percentage = 10 lb_interval_ms = 10000 diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 15a86b84520..cd084fbadfa 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -167,6 +167,7 @@ void info_collector::on_app_stat() all.rdb_block_cache_total_count += row.rdb_block_cache_total_count; all.rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage; all.rdb_memtable_mem_usage += row.rdb_memtable_mem_usage; + all.rdb_estimate_num_keys += row.rdb_estimate_num_keys; read_qps[i] = row.get_qps + row.multi_get_qps + row.scan_qps; write_qps[i] = row.put_qps + row.multi_put_qps + row.remove_qps + row.multi_remove_qps + row.incr_qps + row.check_and_set_qps + row.check_and_mutate_qps; @@ -205,6 +206,7 @@ void info_collector::on_app_stat() counters->rdb_index_and_filter_blocks_mem_usage->set( row.rdb_index_and_filter_blocks_mem_usage); counters->rdb_memtable_mem_usage->set(row.rdb_memtable_mem_usage); + counters->rdb_estimate_num_keys->set(row.rdb_estimate_num_keys); counters->read_qps->set(read_qps[i]); counters->write_qps->set(write_qps[i]); } @@ -255,6 +257,7 @@ info_collector::AppStatCounters *info_collector::get_app_counters(const std::str INIT_COUNTER(rdb_block_cache_hit_rate); INIT_COUNTER(rdb_index_and_filter_blocks_mem_usage); INIT_COUNTER(rdb_memtable_mem_usage); + INIT_COUNTER(rdb_estimate_num_keys); INIT_COUNTER(read_qps); INIT_COUNTER(write_qps); _app_stat_counters[app_name] = counters; diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 69e55e3904b..da4144c3a8c 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -53,6 +53,7 @@ class info_collector ::dsn::perf_counter_wrapper rdb_block_cache_mem_usage; ::dsn::perf_counter_wrapper rdb_index_and_filter_blocks_mem_usage; ::dsn::perf_counter_wrapper rdb_memtable_mem_usage; + ::dsn::perf_counter_wrapper rdb_estimate_num_keys; ::dsn::perf_counter_wrapper read_qps; ::dsn::perf_counter_wrapper write_qps; }; diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 782572e42f1..4827878ea98 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -353,6 +353,13 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) snprintf(name, 255, "rdb.memtable.memory_usage@%s", str_gpid.c_str()); _pfc_rdb_memtable_mem_usage.init_app_counter( "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable"); + + snprintf(name, 255, "rdb.estimate_num_keys@%s", str_gpid.c_str()); + _pfc_rdb_estimate_num_keys.init_app_counter( + "app.pegasus", + name, + COUNTER_TYPE_NUMBER, + "statistics the estimated number of keys inside the rocksdb"); } void pegasus_server_impl::parse_checkpoints() @@ -2287,6 +2294,14 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() _pfc_rdb_memtable_mem_usage->set(val); dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val); } + + // for the same n kv pairs, kEstimateNumKeys will be counted n times, you need compaction to + // remove duplicate + if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) && + dsn::buf2uint64(str_val, val)) { + _pfc_rdb_estimate_num_keys->set(val); + dinfo_replica("_pfc_rdb_estimate_num_keys: {}", val); + } } void pegasus_server_impl::update_server_rocksdb_statistics() diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 49e8c7dd3e1..9498695ee9a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -364,6 +364,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count; ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage; ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage; + ::dsn::perf_counter_wrapper _pfc_rdb_estimate_num_keys; }; } // namespace server diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index e67cee1d978..a4878d84e8a 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -538,6 +538,7 @@ struct row_data double rdb_block_cache_total_count = 0; double rdb_index_and_filter_blocks_mem_usage = 0; double rdb_memtable_mem_usage = 0; + double rdb_estimate_num_keys = 0; }; inline bool @@ -589,6 +590,8 @@ update_app_pegasus_perf_counter(row_data &row, const std::string &counter_name, row.rdb_index_and_filter_blocks_mem_usage += value; else if (counter_name == "rdb.memtable.memory_usage") row.rdb_memtable_mem_usage += value; + else if (counter_name == "rdb.estimate_num_keys") + row.rdb_estimate_num_keys += value; else return false; return true;