From 59f66dcbed0921c4a73f789a7a6fec2906aff0dd Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Sun, 25 Jun 2023 16:02:57 +0800 Subject: [PATCH 1/2] 1 --- be/src/runtime/exec_env.h | 4 ++-- be/src/runtime/exec_env_init.cpp | 8 ++++++-- be/src/runtime/memory/mem_tracker.cpp | 3 ++- be/src/runtime/memory/mem_tracker_limiter.cpp | 3 ++- be/src/vec/exprs/vexpr_context.cpp | 2 -- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 056f2ca125a4bd..157bc5b6f88451 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -217,8 +217,8 @@ class ExecEnv { MemTrackerLimiter* _orphan_mem_tracker_raw; std::shared_ptr _experimental_mem_tracker; // page size not in cache, data page/index page/etc. - std::shared_ptr _page_no_cache_mem_tracker; - std::shared_ptr _brpc_iobuf_block_memory_tracker; + std::unique_ptr _page_no_cache_mem_tracker; + std::unique_ptr _brpc_iobuf_block_memory_tracker; std::unique_ptr _send_batch_thread_pool; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 5d977e1004d461..a1a2432c055c83 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -341,9 +341,9 @@ void ExecEnv::init_mem_tracker() { _experimental_mem_tracker = std::make_shared( MemTrackerLimiter::Type::EXPERIMENTAL, "ExperimentalSet"); _page_no_cache_mem_tracker = - std::make_shared("PageNoCache", _orphan_mem_tracker_raw); + std::make_unique("PageNoCache", _orphan_mem_tracker_raw); _brpc_iobuf_block_memory_tracker = - std::make_shared("IOBufBlockMemory", _orphan_mem_tracker_raw); + std::make_unique("IOBufBlockMemory", _orphan_mem_tracker_raw); } void ExecEnv::init_download_cache_buf() { @@ -424,6 +424,10 @@ void ExecEnv::_destroy() { _join_node_thread_pool.reset(nullptr); _serial_download_cache_thread_token.reset(nullptr); _download_cache_thread_pool.reset(nullptr); + _orphan_mem_tracker.reset(); + _experimental_mem_tracker.reset(); + _page_no_cache_mem_tracker.reset(nullptr); + _brpc_iobuf_block_memory_tracker.reset(nullptr); _is_init = false; } diff --git a/be/src/runtime/memory/mem_tracker.cpp b/be/src/runtime/memory/mem_tracker.cpp index bf3902821dbe1c..7ec943090d0441 100644 --- a/be/src/runtime/memory/mem_tracker.cpp +++ b/be/src/runtime/memory/mem_tracker.cpp @@ -24,6 +24,7 @@ #include +#include "common/daemon.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" @@ -81,7 +82,7 @@ void MemTracker::bind_parent(MemTrackerLimiter* parent) { } MemTracker::~MemTracker() { - if (_parent_group_num != -1) { + if (_parent_group_num != -1 && !k_doris_exit) { std::lock_guard l(mem_tracker_pool[_parent_group_num].group_lock); if (_tracker_group_it != mem_tracker_pool[_parent_group_num].trackers.end()) { mem_tracker_pool[_parent_group_num].trackers.erase(_tracker_group_it); diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index ded5b0c0b8f9ac..d03bd1ac005f12 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -26,6 +26,7 @@ #include #include +#include "common/daemon.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/load_channel_mgr.h" @@ -79,7 +80,7 @@ MemTrackerLimiter::~MemTrackerLimiter() { // in real time. Merge its consumption into orphan when parent is process, to avoid repetition. ExecEnv::GetInstance()->orphan_mem_tracker()->consume(_consumption->current_value()); _consumption->set(0); - { + if (!k_doris_exit) { std::lock_guard l(mem_tracker_limiter_pool[_group_num].group_lock); if (_tracker_limiter_group_it != mem_tracker_limiter_pool[_group_num].trackers.end()) { mem_tracker_limiter_pool[_group_num].trackers.erase(_tracker_limiter_group_it); diff --git a/be/src/vec/exprs/vexpr_context.cpp b/be/src/vec/exprs/vexpr_context.cpp index 5a309a0d4941a3..236dfc404531ff 100644 --- a/be/src/vec/exprs/vexpr_context.cpp +++ b/be/src/vec/exprs/vexpr_context.cpp @@ -21,8 +21,6 @@ #include #include -#include "common/daemon.h" - // IWYU pragma: no_include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/object_pool.h" From 8590fe290750e86a813ea689aeae83ac4ff998c4 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Sun, 25 Jun 2023 17:55:45 +0800 Subject: [PATCH 2/2] 2 --- be/src/runtime/exec_env.h | 4 ++-- be/src/runtime/exec_env_init.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 157bc5b6f88451..056f2ca125a4bd 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -217,8 +217,8 @@ class ExecEnv { MemTrackerLimiter* _orphan_mem_tracker_raw; std::shared_ptr _experimental_mem_tracker; // page size not in cache, data page/index page/etc. - std::unique_ptr _page_no_cache_mem_tracker; - std::unique_ptr _brpc_iobuf_block_memory_tracker; + std::shared_ptr _page_no_cache_mem_tracker; + std::shared_ptr _brpc_iobuf_block_memory_tracker; std::unique_ptr _send_batch_thread_pool; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index a1a2432c055c83..8875cf5463431b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -341,9 +341,9 @@ void ExecEnv::init_mem_tracker() { _experimental_mem_tracker = std::make_shared( MemTrackerLimiter::Type::EXPERIMENTAL, "ExperimentalSet"); _page_no_cache_mem_tracker = - std::make_unique("PageNoCache", _orphan_mem_tracker_raw); + std::make_shared("PageNoCache", _orphan_mem_tracker_raw); _brpc_iobuf_block_memory_tracker = - std::make_unique("IOBufBlockMemory", _orphan_mem_tracker_raw); + std::make_shared("IOBufBlockMemory", _orphan_mem_tracker_raw); } void ExecEnv::init_download_cache_buf() { @@ -426,8 +426,8 @@ void ExecEnv::_destroy() { _download_cache_thread_pool.reset(nullptr); _orphan_mem_tracker.reset(); _experimental_mem_tracker.reset(); - _page_no_cache_mem_tracker.reset(nullptr); - _brpc_iobuf_block_memory_tracker.reset(nullptr); + _page_no_cache_mem_tracker.reset(); + _brpc_iobuf_block_memory_tracker.reset(); _is_init = false; }