fix memtracker bugs during stress test on graphd and storaged (#5276)

* fix memtracker bugs during stress test on graphd and storaged * fix lint * fix RocksEngine memory leak of raw pointer iter * add ENABLE_MEMORY_TRACKER build option & support adaptive limit for MemoryTracker * delete debug log * refine log * refine log * fix build * refine error log * print warning if memtracker is off * fix rocksdb leak by turn off memcheck * refine synamic-self-adaptive * fix cmake check * minor * minor * minor * minor * minor * refine double equel compare
vesoft-inc · Jan 29, 2023 · cfb503e · cfb503e
1 parent 69ee505
commit cfb503e
Show file tree

Hide file tree

Showing 71 changed files with 1,046 additions and 585 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -132,6 +132,7 @@ jobs:
                 -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/clang++ \
                 -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/clang \
                 -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                -DENABLE_MEMORY_TRACKER=off \
                 -DENABLE_ASAN=on \
                 -DENABLE_TESTING=on \
                 -GNinja \

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -65,6 +65,21 @@ if (ENABLE_NATIVE)
     add_compile_options(-fPIC)
 endif()
 
+if(ENABLE_MEMORY_TRACKER)
+    if(ENABLE_JEMALLOC)
+        if(NOT ENABLE_ASAN)
+            add_definitions(-DENABLE_MEMORY_TRACKER)
+            message(STATUS "MemoryTracker is ENABLED")
+        else()
+            message(FATAL_ERROR "MemoryTracker need -DENABLE_ASAN=off")
+        endif()
+    else()
+        message(FATAL_ERROR "MemoryTracker need -DENABLE_JEMALLOC=on")
+    endif()
+else()
+    message(WARNING "MemoryTracker is DISABLED")
+endif()
+
 include_directories(AFTER ${CMAKE_SOURCE_DIR}/src)
 include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src)
 

diff --git a/cmake/nebula/GeneralCMakeOptions.cmake b/cmake/nebula/GeneralCMakeOptions.cmake
@@ -20,6 +20,7 @@ option(ENABLE_GDB_SCRIPT_SECTION        "Add .debug_gdb_scripts section" OFF)
 option(DISABLE_CXX11_ABI                "Whether to disable cxx11 abi" OFF)
 option(ENABLE_BREAKPAD                  "Whether to enable breakpad" OFF)
 option(ENABLE_STANDALONE_VERSION        "Enable standalone version build" OFF)
+option(ENABLE_MEMORY_TRACKER            "Enable memory tracker" ON)
 
 get_cmake_property(variable_list VARIABLES)
 foreach(_varname ${variable_list})

diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h
@@ -76,6 +76,7 @@ StorageClientBase<ClientType, ClientManagerType>::collectResponse(
     folly::EventBase* evb,
     std::unordered_map<HostAddr, Request> requests,
     RemoteFunc&& remoteFunc) {
+  memory::MemoryCheckOffGuard offGuard;
   std::vector<folly::Future<StatusOr<Response>>> respFutures;
   respFutures.reserve(requests.size());
 

diff --git a/src/codec/RowReader.cpp b/src/codec/RowReader.cpp
@@ -15,7 +15,7 @@ namespace nebula {
  * class RowReader::Cell
  *
  ********************************************/
-Value RowReader::Cell::value() const noexcept {
+Value RowReader::Cell::value() const {
   return iter_->reader_->getValueByIndex(iter_->index_);
 }
 
@@ -50,7 +50,7 @@ RowReader::Iterator& RowReader::Iterator::operator++() {
  *
  ********************************************/
 
-bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept {
+bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) {
   schema_ = schema;
   data_ = row;
 

diff --git a/src/codec/RowReader.h b/src/codec/RowReader.h
@@ -25,7 +25,7 @@ class RowReader {
     friend class Iterator;
 
    public:
-    Value value() const noexcept;
+    Value value() const;
 
    private:
     const Iterator* iter_;
@@ -74,15 +74,15 @@ class RowReader {
    * @param prop Property name
    * @return Value Property value
    */
-  virtual Value getValueByName(const std::string& prop) const noexcept = 0;
+  virtual Value getValueByName(const std::string& prop) const = 0;
 
   /**
    * @brief Get the property value by index in schema
    *
    * @param index Index in Schema
    * @return Value Property value
    */
-  virtual Value getValueByIndex(const int64_t index) const noexcept = 0;
+  virtual Value getValueByIndex(const int64_t index) const = 0;
 
   /**
    * @brief Get the timestamp in value
@@ -108,7 +108,7 @@ class RowReader {
    *
    * @return Iterator
    */
-  virtual Iterator begin() const noexcept {
+  virtual Iterator begin() const {
     return Iterator(this, 0);
   }
 
@@ -117,7 +117,7 @@ class RowReader {
    *
    * @return const Iterator&
    */
-  virtual const Iterator& end() const noexcept {
+  virtual const Iterator& end() const {
     return endIter_;
   }
 
@@ -170,7 +170,7 @@ class RowReader {
    * @param row
    * @return Whether reset succeed
    */
-  virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept;
+  virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row);
 
  private:
   Iterator endIter_;

diff --git a/src/codec/RowReaderV1.cpp b/src/codec/RowReaderV1.cpp
@@ -27,7 +27,7 @@ using nebula::cpp2::PropertyType;
  * class RowReaderV1
  *
  ********************************************/
-bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept {
+bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) {
   RowReader::resetImpl(schema, row);
 
   DCHECK(schema_ != nullptr) << "A schema must be provided";
@@ -95,7 +95,7 @@ bool RowReaderV1::processHeader(folly::StringPiece row) {
   return true;
 }
 
-int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept {
+int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const {
   const PropertyType& vType = getSchema()->getFieldType(index);
   if (offsets_[index + 1] >= 0) {
     return offsets_[index + 1];
@@ -160,7 +160,7 @@ int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept {
   return offset;
 }
 
-int64_t RowReaderV1::skipToField(int64_t index) const noexcept {
+int64_t RowReaderV1::skipToField(int64_t index) const {
   DCHECK_GE(index, 0);
   if (index >= static_cast<int64_t>(schema_->getNumFields())) {
     // Index is out of range
@@ -191,12 +191,12 @@ int64_t RowReaderV1::skipToField(int64_t index) const noexcept {
  *  Get the property value
  *
  ***********************************************************/
-Value RowReaderV1::getValueByName(const std::string& prop) const noexcept {
+Value RowReaderV1::getValueByName(const std::string& prop) const {
   int64_t index = getSchema()->getFieldIndex(prop);
   return getValueByIndex(index);
 }
 
-Value RowReaderV1::getValueByIndex(const int64_t index) const noexcept {
+Value RowReaderV1::getValueByIndex(const int64_t index) const {
   if (index < 0 || static_cast<size_t>(index) >= schema_->getNumFields()) {
     return Value(NullType::UNKNOWN_PROP);
   }
@@ -233,7 +233,7 @@ int64_t RowReaderV1::getTimestamp() const noexcept {
  *  Get the property value from the serialized binary string
  *
  ***********************************************************/
-Value RowReaderV1::getBool(int64_t index) const noexcept {
+Value RowReaderV1::getBool(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   switch (getSchema()->getFieldType(index)) {
@@ -272,7 +272,7 @@ Value RowReaderV1::getBool(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getInt(int64_t index) const noexcept {
+Value RowReaderV1::getInt(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   switch (getSchema()->getFieldType(index)) {
@@ -296,7 +296,7 @@ Value RowReaderV1::getInt(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getFloat(int64_t index) const noexcept {
+Value RowReaderV1::getFloat(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   switch (getSchema()->getFieldType(index)) {
@@ -334,7 +334,7 @@ Value RowReaderV1::getFloat(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getDouble(int64_t index) const noexcept {
+Value RowReaderV1::getDouble(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   switch (getSchema()->getFieldType(index)) {
@@ -368,7 +368,7 @@ Value RowReaderV1::getDouble(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getString(int64_t index) const noexcept {
+Value RowReaderV1::getString(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   switch (getSchema()->getFieldType(index)) {
@@ -391,7 +391,7 @@ Value RowReaderV1::getString(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getInt64(int64_t index) const noexcept {
+Value RowReaderV1::getInt64(int64_t index) const {
   RR_GET_OFFSET()
   Value v;
   int64_t val;
@@ -425,7 +425,7 @@ Value RowReaderV1::getInt64(int64_t index) const noexcept {
   return v;
 }
 
-Value RowReaderV1::getVid(int64_t index) const noexcept {
+Value RowReaderV1::getVid(int64_t index) const {
   auto fieldType = getSchema()->getFieldType(index);
   if (fieldType == PropertyType::INT64 || fieldType == PropertyType::VID) {
     // Since 2.0, vid has been defined as a binary array. So we need to convert
@@ -445,7 +445,7 @@ Value RowReaderV1::getVid(int64_t index) const noexcept {
  *  Low-level functions to read from the bytes
  *
  ***********************************************************/
-int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept {
+int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const {
   const uint8_t* start = reinterpret_cast<const uint8_t*>(&(buffer_[offset]));
   folly::ByteRange range(start, buffer_.size() - offset);
 
@@ -457,7 +457,7 @@ int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept {
   return range.begin() - start;
 }
 
-int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept {
+int32_t RowReaderV1::readFloat(int64_t offset, float& v) const {
   if (offset + sizeof(float) > buffer_.size()) {
     return -1;
   }
@@ -467,7 +467,7 @@ int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept {
   return sizeof(float);
 }
 
-int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept {
+int32_t RowReaderV1::readDouble(int64_t offset, double& v) const {
   if (offset + sizeof(double) > buffer_.size()) {
     return -1;
   }
@@ -477,7 +477,7 @@ int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept {
   return sizeof(double);
 }
 
-int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noexcept {
+int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const {
   int64_t strLen = 0;
   int32_t intLen = readInteger(offset, strLen);
   CHECK_GT(intLen, 0) << "Invalid string length";
@@ -489,7 +489,7 @@ int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noe
   return intLen + strLen;
 }
 
-int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept {
+int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const {
   if (offset + sizeof(int64_t) > buffer_.size()) {
     return -1;
   }
@@ -500,7 +500,7 @@ int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept {
   return sizeof(int64_t);
 }
 
-int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const noexcept {
+int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const {
   return readInt64(offset, v);
 }
 

diff --git a/src/codec/RowReaderV1.h b/src/codec/RowReaderV1.h
@@ -27,8 +27,8 @@ class RowReaderV1 : public RowReader {
  public:
   ~RowReaderV1() = default;
 
-  Value getValueByName(const std::string& prop) const noexcept override;
-  Value getValueByIndex(const int64_t index) const noexcept override;
+  Value getValueByName(const std::string& prop) const override;
+  Value getValueByIndex(const int64_t index) const override;
   int64_t getTimestamp() const noexcept override;
 
   int32_t readerVer() const noexcept override {
@@ -40,7 +40,7 @@ class RowReaderV1 : public RowReader {
   }
 
  protected:
-  bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override;
+  bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override;
 
  private:
   int32_t headerLen_ = 0;
@@ -72,31 +72,31 @@ class RowReaderV1 : public RowReader {
   // When succeeded, the method returns the offset pointing to the
   // next field
   // When failed, the method returns a negative number
-  int64_t skipToNext(int64_t index, int64_t offset) const noexcept;
+  int64_t skipToNext(int64_t index, int64_t offset) const;
 
   // Skip to the {index}Th field
   // The method returns the offset of the field
   // It returns a negative number when the data corrupts
-  int64_t skipToField(int64_t index) const noexcept;
+  int64_t skipToField(int64_t index) const;
 
   // Following methods assume the parameters index are valid
   // When succeeded, offset will advance
-  Value getBool(int64_t index) const noexcept;
-  Value getInt(int64_t index) const noexcept;
-  Value getFloat(int64_t index) const noexcept;
-  Value getDouble(int64_t index) const noexcept;
-  Value getString(int64_t index) const noexcept;
-  Value getInt64(int64_t index) const noexcept;
-  Value getVid(int64_t index) const noexcept;
+  Value getBool(int64_t index) const;
+  Value getInt(int64_t index) const;
+  Value getFloat(int64_t index) const;
+  Value getDouble(int64_t index) const;
+  Value getString(int64_t index) const;
+  Value getInt64(int64_t index) const;
+  Value getVid(int64_t index) const;
 
   // The following methods all return the number of bytes read
   // A negative number will be returned if an error occurs
-  int32_t readInteger(int64_t offset, int64_t& v) const noexcept;
-  int32_t readFloat(int64_t offset, float& v) const noexcept;
-  int32_t readDouble(int64_t offset, double& v) const noexcept;
-  int32_t readString(int64_t offset, folly::StringPiece& v) const noexcept;
-  int32_t readInt64(int64_t offset, int64_t& v) const noexcept;
-  int32_t readVid(int64_t offset, int64_t& v) const noexcept;
+  int32_t readInteger(int64_t offset, int64_t& v) const;
+  int32_t readFloat(int64_t offset, float& v) const;
+  int32_t readDouble(int64_t offset, double& v) const;
+  int32_t readString(int64_t offset, folly::StringPiece& v) const;
+  int32_t readInt64(int64_t offset, int64_t& v) const;
+  int32_t readVid(int64_t offset, int64_t& v) const;
 };
 
 }  // namespace nebula

diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp
@@ -9,7 +9,7 @@ namespace nebula {
 
 using nebula::cpp2::PropertyType;
 
-bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept {
+bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) {
   RowReader::resetImpl(schema, row);
 
   DCHECK(!!schema_);
@@ -45,12 +45,12 @@ bool RowReaderV2::isNull(size_t pos) const {
   return flag != 0;
 }
 
-Value RowReaderV2::getValueByName(const std::string& prop) const noexcept {
+Value RowReaderV2::getValueByName(const std::string& prop) const {
   int64_t index = schema_->getFieldIndex(prop);
   return getValueByIndex(index);
 }
 
-Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept {
+Value RowReaderV2::getValueByIndex(const int64_t index) const {
   if (index < 0 || static_cast<size_t>(index) >= schema_->getNumFields()) {
     return Value(NullType::UNKNOWN_PROP);
   }

diff --git a/src/codec/RowReaderV2.h b/src/codec/RowReaderV2.h
@@ -28,8 +28,8 @@ class RowReaderV2 : public RowReader {
  public:
   ~RowReaderV2() override = default;
 
-  Value getValueByName(const std::string& prop) const noexcept override;
-  Value getValueByIndex(const int64_t index) const noexcept override;
+  Value getValueByName(const std::string& prop) const override;
+  Value getValueByIndex(const int64_t index) const override;
   int64_t getTimestamp() const noexcept override;
 
   int32_t readerVer() const noexcept override {
@@ -41,7 +41,7 @@ class RowReaderV2 : public RowReader {
   }
 
  protected:
-  bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override;
+  bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override;
 
  private:
   size_t headerLen_;