Merge branch 'master' into fix-VertexHolder-getDefaultProp

etc/nebula-storaged.conf.default

@@ -65,6 +65,17 @@
 # In order to disable compression for level 0/1, set it to "no:no"
 --rocksdb_compression_per_level=
 
+# Whether or not to enable rocksdb's statistics, disabled by default
+--enable_rocksdb_statistics=false
+
+# Statslevel used by rocksdb to collection statistics, optional values are
+#   * kExceptHistogramOrTimers, disable timer stats, and skip histogram stats
+#   * kExceptTimers, Skip timer stats
+#   * kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression.
+#   * kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock.
+#   * kAll, Collect all stats
+--rocksdb_stats_level=kExceptHistogramOrTimers
+
 ############## rocksdb Options ##############
 # rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma
 --rocksdb_db_options={"max_subcompactions":"1","max_background_jobs":"1"}

etc/nebula-storaged.conf.production

@@ -73,6 +73,17 @@
 # rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma
 --rocksdb_block_based_table_options={"block_size":"8192"}
 
+# Whether or not to enable rocksdb's statistics, disabled by default
+--enable_rocksdb_statistics=false
+
+# Statslevel used by rocksdb to collection statistics, optional values are
+#   * kExceptHistogramOrTimers, disable timer stats, and skip histogram stats
+#   * kExceptTimers, Skip timer stats
+#   * kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression.
+#   * kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock.
+#   * kAll, Collect all stats
+--rocksdb_stats_level=kExceptHistogramOrTimers
+
 ############### misc ####################
 --max_handlers_per_req=1
 --heartbeat_interval_secs=10

src/kvstore/RocksEngineConfig.cpp

@@ -60,6 +60,9 @@ DEFINE_string(rocksdb_compression_per_level, "", "Specify per level compression
                                                  "e.g. \"no:no:lz4:lz4::zstd\" === "
                                                  "\"no:no:lz4:lz4:lz4:snappy:zstd:snappy\"");
 
+DEFINE_bool(enable_rocksdb_statistics, false, "Whether or not to enable rocksdb's statistics");
+DEFINE_string(rocksdb_stats_level, "kExceptHistogramOrTimers", "rocksdb statistics level");
+
 namespace nebula {
 namespace kvstore {
 
@@ -123,6 +126,10 @@ rocksdb::Status initRocksdbOptions(rocksdb::Options &baseOpts) {
     if (!s.ok()) {
         return s;
     }
+    std::shared_ptr<rocksdb::Statistics> stats = getDBStatistics();
+    if (stats) {
+        dbOpts.statistics = std::move(stats);
+    }
     dbOpts.listeners.emplace_back(new EventListener());
 
     std::unordered_map<std::string, std::string> cfOptsMap;
@@ -186,5 +193,29 @@ bool loadOptionsMap(std::unordered_map<std::string, std::string> &map, const std
     return true;
 }
 
+static std::shared_ptr<rocksdb::Statistics> createDBStatistics() {
+    std::shared_ptr<rocksdb::Statistics> dbstats = rocksdb::CreateDBStatistics();
+    if (FLAGS_rocksdb_stats_level == "kExceptHistogramOrTimers") {
+        dbstats->set_stats_level(rocksdb::StatsLevel::kExceptHistogramOrTimers);
+    } else if (FLAGS_rocksdb_stats_level == "kExceptTimers") {
+        dbstats->set_stats_level(rocksdb::StatsLevel::kExceptTimers);
+    } else if (FLAGS_rocksdb_stats_level == "kExceptDetailedTimers") {
+        dbstats->set_stats_level(rocksdb::StatsLevel::kExceptDetailedTimers);
+    } else if (FLAGS_rocksdb_stats_level == "kExceptTimeForMutex") {
+        dbstats->set_stats_level(rocksdb::StatsLevel::kExceptTimeForMutex);
+    } else {
+        dbstats->set_stats_level(rocksdb::StatsLevel::kAll);
+    }
+    return dbstats;
+}
+
+std::shared_ptr<rocksdb::Statistics> getDBStatistics() {
+    if (FLAGS_enable_rocksdb_statistics) {
+        static std::shared_ptr<rocksdb::Statistics> dbstats = createDBStatistics();
+        return dbstats;
+    }
+    return nullptr;
+}
+
 }  // namespace kvstore
 }  // namespace nebula

src/kvstore/RocksEngineConfig.h

@@ -40,6 +40,8 @@ DECLARE_string(part_man_type);
 DECLARE_string(rocksdb_compression_per_level);
 DECLARE_string(rocksdb_compression);
 
+DECLARE_bool(enable_rocksdb_statistics);
+DECLARE_string(rocksdb_stats_level);
 
 namespace nebula {
 namespace kvstore {
@@ -48,6 +50,8 @@ rocksdb::Status initRocksdbOptions(rocksdb::Options &baseOpts);
 
 bool loadOptionsMap(std::unordered_map<std::string, std::string> &map, const std::string& gflags);
 
+std::shared_ptr<rocksdb::Statistics> getDBStatistics();
+
 }  // namespace kvstore
 }  // namespace nebula
 #endif  // KVSTORE_ROCKSENGINECONFIG_H_

src/kvstore/test/RocksEngineConfigTest.cpp

@@ -88,6 +88,25 @@ TEST(RocksEngineConfigTest, createOptionsTest) {
     FLAGS_rocksdb_db_options = "{}";
 }
 
+TEST(RocksEngineConfigTest, StatisticsConfigTest) {
+    {
+        FLAGS_enable_rocksdb_statistics = false;
+        rocksdb::Options options;
+        auto status = initRocksdbOptions(options);
+        ASSERT_TRUE(status.ok()) << status.ToString();
+        ASSERT_EQ(nullptr, getDBStatistics());
+    }
+
+    {
+        FLAGS_enable_rocksdb_statistics = true;
+        FLAGS_rocksdb_stats_level = "kExceptTimers";
+        rocksdb::Options options;
+        auto status = initRocksdbOptions(options);
+        ASSERT_TRUE(status.ok()) << status.ToString();
+        std::shared_ptr<rocksdb::Statistics> stats = getDBStatistics();
+        ASSERT_EQ(rocksdb::StatsLevel::kExceptTimers, stats->get_stats_level());
+    }
+}
 
 TEST(RocksEngineConfigTest, CompressionConfigTest) {
     {

src/kvstore/wal/FileBasedWal.cpp

@@ -60,7 +60,10 @@ FileBasedWal::FileBasedWal(const folly::StringPiece dir,
                   << ", path is " << info->path();
         currFd_ = open(info->path(), O_WRONLY | O_APPEND);
         currInfo_ = info;
-        CHECK_GE(currFd_, 0);
+        if (currFd_ < 0) {
+            LOG(FATAL) << "Failed to open the file \"" << info->path() << "\" ("
+                       << errno << "): " << strerror(errno);
+        }
     }
 }