From c6d6f0d776210747d32cf420c5750da8f9d4c8dc Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Fri, 29 Nov 2019 15:46:17 +0800 Subject: [PATCH 01/10] add key number --- src/server/pegasus_server_impl.cpp | 10 ++++++ src/server/pegasus_server_impl.h | 1 + src/server/test/pegasus_server_impl_test.cpp | 34 +++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 39b96b6d0e..c23a03db3d 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -338,6 +338,10 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) snprintf(name, 255, "rdb.memtable.memory_usage@%s", str_gpid.c_str()); _pfc_rdb_memtable_mem_usage.init_app_counter( "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable"); + + snprintf(name, 255, "rdb.estimate.key_number@%s", str_gpid.c_str()); + _pfc_rdb_estimate_key_number.init_app_counter( + "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the key number of rocksdb all data"); } void pegasus_server_impl::parse_checkpoints() @@ -2242,6 +2246,12 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() _pfc_rdb_memtable_mem_usage->set(val); dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val); } + + if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) && + dsn::buf2uint64(str_val, val)) { + _pfc_rdb_estimate_key_number->set(val); + dinfo_replica("_pfc_rdb_estimate_key_number: {}", val); + } } void pegasus_server_impl::update_server_rocksdb_statistics() diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 49e8c7dd3e..563fdcc535 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -364,6 +364,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count; ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage; ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage; + ::dsn::perf_counter_wrapper _pfc_rdb_estimate_key_number; }; } // namespace server diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 2ad29e9d45..821494c673 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -3,15 +3,23 @@ // can be found in the LICENSE file in the root directory of this source tree. #include +#include +#include "server/pegasus_server_write.h" +#include "server/pegasus_write_service_impl.h" #include "pegasus_server_test_base.h" +#include "message_utils.h" namespace pegasus { namespace server { class pegasus_server_impl_test : public pegasus_server_test_base { + + std::unique_ptr _server_write; + public: - pegasus_server_impl_test() : pegasus_server_test_base() { start(); } + pegasus_server_impl_test() : pegasus_server_test_base() { start(); + _server_write = dsn::make_unique(_server.get(), true);} void test_table_level_slow_query() { @@ -55,9 +63,33 @@ class pegasus_server_impl_test : public pegasus_server_test_base ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count); } } + + void test_table_property() + { + std::unique_ptr _server_write; + _server_write = dsn::make_unique(_server.get(), true); + dsn::blob key; + pegasus_generate_key(key, std::string("hash"), std::string("sort")); + dsn::apps::update_request req; + req.key = key; + req.value.assign("value", 0, 5); + + int put_rpc_cnt = 5; + auto writes = new dsn::message_ex *[put_rpc_cnt]; + for (int i = 0; i < put_rpc_cnt; i++) { + writes[i] = pegasus::create_put_request(req); + } + _server_write->on_batched_write_requests(writes, put_rpc_cnt, 0, 0); + + std::string str_val; + _server->_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val); + ASSERT_EQ(str_val, std::to_string(put_rpc_cnt)); + } }; TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); } +TEST_F(pegasus_server_impl_test, test_table_property){test_table_property();} + } // namespace server } // namespace pegasus From a5f2d4aa030b0cc5082451b868bf3264e6c0af72 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 11:04:26 +0800 Subject: [PATCH 02/10] format --- src/server/pegasus_server_impl.cpp | 2 +- src/server/test/pegasus_server_impl_test.cpp | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index c23a03db3d..7f3589aa66 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -341,7 +341,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) snprintf(name, 255, "rdb.estimate.key_number@%s", str_gpid.c_str()); _pfc_rdb_estimate_key_number.init_app_counter( - "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the key number of rocksdb all data"); + "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the key number of rocksdb all data"); } void pegasus_server_impl::parse_checkpoints() diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 821494c673..e50793044b 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -18,8 +18,11 @@ class pegasus_server_impl_test : public pegasus_server_test_base std::unique_ptr _server_write; public: - pegasus_server_impl_test() : pegasus_server_test_base() { start(); - _server_write = dsn::make_unique(_server.get(), true);} + pegasus_server_impl_test() : pegasus_server_test_base() + { + start(); + _server_write = dsn::make_unique(_server.get(), true); + } void test_table_level_slow_query() { @@ -89,7 +92,7 @@ class pegasus_server_impl_test : public pegasus_server_test_base TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); } -TEST_F(pegasus_server_impl_test, test_table_property){test_table_property();} +TEST_F(pegasus_server_impl_test, test_table_property) { test_table_property(); } } // namespace server } // namespace pegasus From 533a1a70c0aedf51b81e04681e2b66d0a19de636 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 13:56:10 +0800 Subject: [PATCH 03/10] format and fix --- src/server/pegasus_server_impl.cpp | 7 ++- src/server/test/pegasus_server_impl_test.cpp | 55 +++++++++----------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 7f3589aa66..ca8083539b 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -339,9 +339,12 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _pfc_rdb_memtable_mem_usage.init_app_counter( "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable"); - snprintf(name, 255, "rdb.estimate.key_number@%s", str_gpid.c_str()); + snprintf(name, 255, "rdb.estimate_key_number@%s", str_gpid.c_str()); _pfc_rdb_estimate_key_number.init_app_counter( - "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the key number of rocksdb all data"); + "app.pegasus", + name, + COUNTER_TYPE_NUMBER, + "statistic the estimate key number of rocksdb all data"); } void pegasus_server_impl::parse_checkpoints() diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index e50793044b..067a7e7d0f 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -15,14 +15,10 @@ namespace server { class pegasus_server_impl_test : public pegasus_server_test_base { - std::unique_ptr _server_write; - public: - pegasus_server_impl_test() : pegasus_server_test_base() - { - start(); - _server_write = dsn::make_unique(_server.get(), true); - } + pegasus_server_impl_test() : pegasus_server_test_base() { start(); } + + rocksdb::DB *get_server_db() { return _server->_db; } void test_table_level_slow_query() { @@ -66,33 +62,32 @@ class pegasus_server_impl_test : public pegasus_server_test_base ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count); } } - - void test_table_property() - { - std::unique_ptr _server_write; - _server_write = dsn::make_unique(_server.get(), true); - dsn::blob key; - pegasus_generate_key(key, std::string("hash"), std::string("sort")); - dsn::apps::update_request req; - req.key = key; - req.value.assign("value", 0, 5); - - int put_rpc_cnt = 5; - auto writes = new dsn::message_ex *[put_rpc_cnt]; - for (int i = 0; i < put_rpc_cnt; i++) { - writes[i] = pegasus::create_put_request(req); - } - _server_write->on_batched_write_requests(writes, put_rpc_cnt, 0, 0); - - std::string str_val; - _server->_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val); - ASSERT_EQ(str_val, std::to_string(put_rpc_cnt)); - } }; TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); } -TEST_F(pegasus_server_impl_test, test_table_property) { test_table_property(); } +TEST_F(pegasus_server_impl_test, test_rdb_estimate_num_keys) +{ + + std::unique_ptr server_write = + dsn::make_unique(_server.get(), true); + + dsn::apps::update_request req; + pegasus_generate_key(req.key, std::string("hash"), std::string("sort")); + req.value.assign("value", 0, 5); + + const int put_cnt = 5; + auto writes = new dsn::message_ex *[put_cnt]; + for (int i = 0; i < put_cnt; i++) { + writes[i] = pegasus::create_put_request(req); + } + server_write->on_batched_write_requests(writes, put_cnt, 0, 0); + auto cleanup = dsn::defer([=]() { delete[] writes; }); + + std::string str_val; + get_server_db()->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val); + ASSERT_EQ(atoi(str_val.c_str()), put_cnt); +} } // namespace server } // namespace pegasus From 9f5d9cc27c41bd4bc357087a0cc009f2febbd801 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 13:58:25 +0800 Subject: [PATCH 04/10] format and fix --- src/server/test/pegasus_server_impl_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 067a7e7d0f..30b1d89b55 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -14,7 +14,6 @@ namespace server { class pegasus_server_impl_test : public pegasus_server_test_base { - public: pegasus_server_impl_test() : pegasus_server_test_base() { start(); } From 943e18b1e21e7ac7d336aa75ab1e3e453d02ceeb Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 14:03:36 +0800 Subject: [PATCH 05/10] format and fix --- src/server/test/pegasus_server_impl_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 30b1d89b55..6de04d32d9 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -67,7 +67,6 @@ TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level TEST_F(pegasus_server_impl_test, test_rdb_estimate_num_keys) { - std::unique_ptr server_write = dsn::make_unique(_server.get(), true); @@ -77,6 +76,7 @@ TEST_F(pegasus_server_impl_test, test_rdb_estimate_num_keys) const int put_cnt = 5; auto writes = new dsn::message_ex *[put_cnt]; + // generate same five kv, and the kEstimateNumKeys will be count five time for (int i = 0; i < put_cnt; i++) { writes[i] = pegasus::create_put_request(req); } From 8658b539f924cb688c0fc571f9294e534fdb4ae3 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 14:27:09 +0800 Subject: [PATCH 06/10] format and fix --- src/server/pegasus_server_impl.cpp | 2 ++ src/server/test/pegasus_server_impl_test.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index ca8083539b..45a4a5c93b 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2250,6 +2250,8 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val); } + // for the same n kv pairs, kEstimateNumKeys will be counted n time, you need compaction to + // remove duplicate if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) && dsn::buf2uint64(str_val, val)) { _pfc_rdb_estimate_key_number->set(val); diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 6de04d32d9..7f7e957394 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -76,12 +76,12 @@ TEST_F(pegasus_server_impl_test, test_rdb_estimate_num_keys) const int put_cnt = 5; auto writes = new dsn::message_ex *[put_cnt]; - // generate same five kv, and the kEstimateNumKeys will be count five time + auto cleanup = dsn::defer([=]() { delete[] writes; }); + // generate same put_cnt kv paris, and the kEstimateNumKeys will be counted put_cnt times for (int i = 0; i < put_cnt; i++) { writes[i] = pegasus::create_put_request(req); } server_write->on_batched_write_requests(writes, put_cnt, 0, 0); - auto cleanup = dsn::defer([=]() { delete[] writes; }); std::string str_val; get_server_db()->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val); From ca6d634266bf3fa3b216dba5c77b8f4b162edb90 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 14:29:20 +0800 Subject: [PATCH 07/10] format and fix --- src/server/pegasus_server_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 45a4a5c93b..691979cf0e 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2250,7 +2250,7 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val); } - // for the same n kv pairs, kEstimateNumKeys will be counted n time, you need compaction to + // for the same n kv pairs, kEstimateNumKeys will be counted n times, you need compaction to // remove duplicate if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) && dsn::buf2uint64(str_val, val)) { From 8dc8229f2a007b585026b3a27617ec0c1ec82961 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 17:30:00 +0800 Subject: [PATCH 08/10] format and fix --- src/server/pegasus_server_impl.cpp | 10 +++++----- src/server/pegasus_server_impl.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 691979cf0e..1b9c4aa200 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -339,12 +339,12 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _pfc_rdb_memtable_mem_usage.init_app_counter( "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable"); - snprintf(name, 255, "rdb.estimate_key_number@%s", str_gpid.c_str()); - _pfc_rdb_estimate_key_number.init_app_counter( + snprintf(name, 255, "rdb.estimate_num_keys@%s", str_gpid.c_str()); + _pfc_rdb_estimate_num_keys.init_app_counter( "app.pegasus", name, COUNTER_TYPE_NUMBER, - "statistic the estimate key number of rocksdb all data"); + "statistic the estimate number keys of rocksdb all data"); } void pegasus_server_impl::parse_checkpoints() @@ -2254,8 +2254,8 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() // remove duplicate if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) && dsn::buf2uint64(str_val, val)) { - _pfc_rdb_estimate_key_number->set(val); - dinfo_replica("_pfc_rdb_estimate_key_number: {}", val); + _pfc_rdb_estimate_num_keys->set(val); + dinfo_replica("_pfc_rdb_estimate_num_keys: {}", val); } } diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 563fdcc535..9498695ee9 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -364,7 +364,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count; ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage; ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage; - ::dsn::perf_counter_wrapper _pfc_rdb_estimate_key_number; + ::dsn::perf_counter_wrapper _pfc_rdb_estimate_num_keys; }; } // namespace server From f9dafccaf1035c1df4240b427f8012a56ac84762 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 18:14:54 +0800 Subject: [PATCH 09/10] format and fix --- src/server/pegasus_server_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 1b9c4aa200..033462b063 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -344,7 +344,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) "app.pegasus", name, COUNTER_TYPE_NUMBER, - "statistic the estimate number keys of rocksdb all data"); + "statistics the estimated number of keys inside the rocksdb"); } void pegasus_server_impl::parse_checkpoints() From e835818d2c1a29918c383dc40f8c41cc6a6d6564 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 3 Dec 2019 18:38:43 +0800 Subject: [PATCH 10/10] format and fix --- src/server/test/pegasus_server_impl_test.cpp | 29 -------------------- 1 file changed, 29 deletions(-) diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 7f7e957394..2ad29e9d45 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -3,11 +3,7 @@ // can be found in the LICENSE file in the root directory of this source tree. #include -#include -#include "server/pegasus_server_write.h" -#include "server/pegasus_write_service_impl.h" #include "pegasus_server_test_base.h" -#include "message_utils.h" namespace pegasus { namespace server { @@ -17,8 +13,6 @@ class pegasus_server_impl_test : public pegasus_server_test_base public: pegasus_server_impl_test() : pegasus_server_test_base() { start(); } - rocksdb::DB *get_server_db() { return _server->_db; } - void test_table_level_slow_query() { // the on_get function will sleep 10ms for unit test, @@ -65,28 +59,5 @@ class pegasus_server_impl_test : public pegasus_server_test_base TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); } -TEST_F(pegasus_server_impl_test, test_rdb_estimate_num_keys) -{ - std::unique_ptr server_write = - dsn::make_unique(_server.get(), true); - - dsn::apps::update_request req; - pegasus_generate_key(req.key, std::string("hash"), std::string("sort")); - req.value.assign("value", 0, 5); - - const int put_cnt = 5; - auto writes = new dsn::message_ex *[put_cnt]; - auto cleanup = dsn::defer([=]() { delete[] writes; }); - // generate same put_cnt kv paris, and the kEstimateNumKeys will be counted put_cnt times - for (int i = 0; i < put_cnt; i++) { - writes[i] = pegasus::create_put_request(req); - } - server_write->on_batched_write_requests(writes, put_cnt, 0, 0); - - std::string str_val; - get_server_db()->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val); - ASSERT_EQ(atoi(str_val.c_str()), put_cnt); -} - } // namespace server } // namespace pegasus