Skip to content

Commit 6808e11

Browse files
authored
[fix](memory) Fix free page not tracking correctly (#36768)
## Proposed changes #36235 introduced, index page is not freed in query thread, so saved tracker ptr when alloc page, so that page only needs to be freed before query ends. ``` F20240624 18:44:51.880192 162249 mem_tracker_limiter.cpp:125] mem tracker label: Query#Id=f0511107829e465e-a87a985234018b77, consumption: 94, peak consumption: 636540, mem tracker not equal to 0 when mem tracker destruct, this usually means that memory tracking is inaccurate and SCOPED_ATTACH_TASK and SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER are not used correctly. 1. For query and load, memory leaks may have occurred, it is expected that the query mem tracker will be bound to the thread context using SCOPED_ATTACH_TASK and SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER before all memory alloc and free. 2. If a memory alloc is recorded by this tracker, it is expected that be recorded in this tracker when memory is freed. 3. Merge the remaining memory tracking value by this tracker into Orphan, if you observe that Orphan is not equal to 0 in the mem tracker web or log, this indicates that there may be a memory leak. 4. If you need to transfer memory tracking value between two trackers, can use transfer_to..[Address Sanitizer]: 0x605000160b30, size 46, strack trace: 0# Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long) 1# doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, bool, doris::segment_v2::PageTypePB) 2# doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*) 3# doris::segment_v2::IndexedColumnReader::read_page(doris::segment_v2::PagePointer const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*, doris::segment_v2::PageTypePB, doris::BlockCompressionCodec*, bool) const 4# doris::segment_v2::IndexedColumnReader::load_index_page(doris::segment_v2::PagePointerPB const&, doris::segment_v2::PageHandle*, doris::segment_v2::IndexPageReader*) 5# doris::segment_v2::IndexedColumnReader::load(bool, bool) 6# doris::segment_v2::BitmapIndexReader::_load(bool, bool, std::unique_ptr<doris::segment_v2::BitmapIndexPB, std::default_delete<doris::segment_v2::BitmapIndexPB> >) 7# doris::segment_v2::BitmapIndexReader::load(bool, bool) 8# doris::segment_v2::ColumnReader::_load_bitmap_index(bool, bool) 9# doris::segment_v2::ColumnReader::new_bitmap_index_iterator(doris::segment_v2::BitmapIndexIterator**) 10# doris::segment_v2::Segment::new_bitmap_index_iterator(doris::TabletColumn const&, std::unique_ptr<doris::segment_v2::BitmapIndexIterator, std::default_delete<doris::segment_v2::BitmapIndexIterator> >*) 11# doris::segment_v2::SegmentIterator::_init_bitmap_index_iterators() 12# doris::segment_v2::SegmentIterator::init_iterators() 13# doris::segment_v2::SegmentIterator::_init_impl(doris::StorageReadOptions const&) 14# doris::segment_v2::SegmentIterator::init(doris::StorageReadOptions const&) 15# doris::segment_v2::Segment::new_iterator(std::shared_ptr<doris::Schema const>, doris::StorageReadOptions const&, std::unique_ptr<doris::RowwiseIterator, std::default_delete<doris::RowwiseIterator> >*) 16# doris::segment_v2::LazyInitSegmentIterator::init(doris::StorageReadOptions const&) 17# doris::BetaRowsetReader::_init_iterator() 18# doris::BetaRowsetReader::_init_iterator_once() 19# doris::BetaRowsetReader::next_block(doris::vectorized::Block*) 20# doris::vectorized::VCollectIterator::_topn_next(doris::vectorized::Block*) 21# doris::vectorized::VCollectIterator::next(doris::vectorized::Block*) 22# doris::vectorized::BlockReader::_direct_next_block(doris::vectorized::Block*, bool*) 23# doris::vectorized::BlockReader::next_block_with_aggregation(doris::vectorized::Block*, bool*) 24# doris::vectorized::NewOlapScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) 25# doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) 26# doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) 27# doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) 28# std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&) 29# doris::ThreadPool::dispatch_thread() 30# doris::Thread::supervise_thread(void*) ```
1 parent f719957 commit 6808e11

File tree

2 files changed

+10
-18
lines changed

2 files changed

+10
-18
lines changed

be/src/olap/page_cache.cpp

+9-16
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,14 @@
2626
namespace doris {
2727
template <typename TAllocator>
2828
PageBase<TAllocator>::PageBase(size_t b, bool use_cache, segment_v2::PageTypePB page_type)
29-
: LRUCacheValueBase(),
30-
_size(b),
31-
_capacity(b),
32-
_use_cache(use_cache),
33-
_page_type(page_type) {
34-
if (_use_cache) {
35-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
36-
StoragePageCache::instance()->mem_tracker(_page_type));
37-
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity, ALLOCATOR_ALIGNMENT_16));
29+
: LRUCacheValueBase(), _size(b), _capacity(b) {
30+
if (use_cache) {
31+
_mem_tracker_by_allocator = StoragePageCache::instance()->mem_tracker(page_type);
3832
} else {
33+
_mem_tracker_by_allocator = thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker();
34+
}
35+
{
36+
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
3937
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity, ALLOCATOR_ALIGNMENT_16));
4038
}
4139
}
@@ -44,13 +42,8 @@ template <typename TAllocator>
4442
PageBase<TAllocator>::~PageBase() {
4543
if (_data != nullptr) {
4644
DCHECK(_capacity != 0 && _size != 0);
47-
if (_use_cache) {
48-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
49-
StoragePageCache::instance()->mem_tracker(_page_type));
50-
TAllocator::free(_data, _capacity);
51-
} else {
52-
TAllocator::free(_data, _capacity);
53-
}
45+
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
46+
TAllocator::free(_data, _capacity);
5447
}
5548
}
5649

be/src/olap/page_cache.h

+1-2
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ class PageBase : private TAllocator, public LRUCacheValueBase {
6060
// Effective size, smaller than capacity, such as data page remove checksum suffix.
6161
size_t _size = 0;
6262
size_t _capacity = 0;
63-
bool _use_cache;
64-
segment_v2::PageTypePB _page_type;
63+
std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
6564
};
6665

6766
using DataPage = PageBase<Allocator<false>>;

0 commit comments

Comments
 (0)