From 27ffc1fbfcbf0cda966e750c8f1ae55548639423 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Mon, 11 Nov 2024 17:23:12 +0800 Subject: [PATCH] [fix](cloud-mow) Add enable config for procesing old version delete bitmap when doing cu compaction --- be/src/cloud/cloud_cumulative_compaction.cpp | 3 ++- be/src/cloud/cloud_delete_bitmap_action.cpp | 2 ++ be/src/cloud/cloud_meta_mgr.cpp | 2 +- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/olap/rowset/beta_rowset_writer.cpp | 3 +-- be/src/olap/tablet_meta.cpp | 3 +++ 7 files changed, 13 insertions(+), 4 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 4eb8aba2068def..92e0f44dc5b44d 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -360,7 +360,8 @@ Status CloudCumulativeCompaction::modify_rowsets() { stats.num_rows(), stats.data_size()); } } - if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && + if (config::enable_delete_bitmap_merge_on_compaction && + _tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { RETURN_IF_ERROR(process_old_version_delete_bitmap()); } diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/cloud/cloud_delete_bitmap_action.cpp index 60db5896dfab8a..672574a5aa8901 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ b/be/src/cloud/cloud_delete_bitmap_action.cpp @@ -95,6 +95,8 @@ Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); auto size = tablet->tablet_meta()->delete_bitmap().get_size(); + LOG(INFO) << "show_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count + << ",cardinality=" << cardinality << ",size=" << size; rapidjson::Document root; root.SetObject(); diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 026610d5b0cd4b..c1440944f7968a 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -1074,7 +1074,7 @@ Status CloudMetaMgr::update_delete_bitmap(const CloudTablet& tablet, int64_t loc Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(const CloudTablet& tablet, DeleteBitmap* delete_bitmap) { LOG(INFO) << "cloud_update_delete_bitmap_without_lock , tablet_id: " << tablet.tablet_id() - << ",delete_bitmap size:" << delete_bitmap->delete_bitmap.size(); + << ",delete_bitmap size:" << delete_bitmap->get_delete_bitmap_count(); UpdateDeleteBitmapRequest req; UpdateDeleteBitmapResponse res; req.set_cloud_unique_id(config::cloud_unique_id); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 6c468e5abde5a0..ef70fd9b88c8f6 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1379,6 +1379,8 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect, "false"); DEFINE_Int32(query_cache_size, "512"); +DEFINE_mBool(enable_delete_bitmap_merge_on_compaction, "false"); + // Enable validation to check the correctness of table size. DEFINE_Bool(enable_table_size_correctness_check, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index c763d1f8a061ba..726835ef85f03a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1459,6 +1459,8 @@ DECLARE_mBool(enable_pipeline_task_leakage_detect); // MB DECLARE_Int32(query_cache_size); +DECLARE_mBool(enable_delete_bitmap_merge_on_compaction); + // Enable validation to check the correctness of table size. DECLARE_Bool(enable_table_size_correctness_check); diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 9cd634ec5aca1e..e46909c96237c3 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -336,8 +336,7 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id() << ", rowset_ids: " << _context.mow_context->rowset_ids.size() << ", cur max_version: " << _context.mow_context->max_version - << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: " - << _context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count() + << ", transaction_id: " << _context.mow_context->txn_id << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows; return Status::OK(); } diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index b5e3accab86b36..9c60eeb3c658cc 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1188,6 +1188,9 @@ void DeleteBitmap::add_to_remove_queue( } void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector& vector) { + if (!config::enable_delete_bitmap_merge_on_compaction) { + return; + } std::shared_lock l(stale_delete_bitmap_lock); // std::vector> to_delete;