Skip to content

Commit 35e0d56

Browse files
dataroaringHappenLee
dataroaring
authored and
HappenLee
committed
[enhance](partition_id) check partition id before store meta (apache#28055)
1 parent 91f1442 commit 35e0d56

18 files changed

+48
-16
lines changed

‎be/src/olap/data_dir.cpp

+13-5
Original file line numberDiff line numberDiff line change
@@ -539,15 +539,23 @@ Status DataDir::load() {
539539
StorageEngine::instance()->pending_local_rowsets().add(
540540
rowset_meta->rowset_id()),
541541
true);
542-
if (!commit_txn_status && !commit_txn_status.is<PUSH_TRANSACTION_ALREADY_EXIST>()) {
543-
LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id()
544-
<< " to tablet: " << rowset_meta->tablet_id()
545-
<< " for txn: " << rowset_meta->txn_id();
546-
} else {
542+
if (commit_txn_status || commit_txn_status.is<PUSH_TRANSACTION_ALREADY_EXIST>()) {
547543
LOG(INFO) << "successfully to add committed rowset: " << rowset_meta->rowset_id()
548544
<< " to tablet: " << rowset_meta->tablet_id()
549545
<< " schema hash: " << rowset_meta->tablet_schema_hash()
550546
<< " for txn: " << rowset_meta->txn_id();
547+
548+
} else if (commit_txn_status.is<ErrorCode::INTERNAL_ERROR>()) {
549+
LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id()
550+
<< " to tablet: " << rowset_meta->tablet_id()
551+
<< " for txn: " << rowset_meta->txn_id()
552+
<< " error: " << commit_txn_status;
553+
return commit_txn_status;
554+
} else {
555+
LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id()
556+
<< " to tablet: " << rowset_meta->tablet_id()
557+
<< " for txn: " << rowset_meta->txn_id()
558+
<< " error: " << commit_txn_status;
551559
}
552560
} else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE &&
553561
rowset_meta->tablet_uid() == tablet->tablet_uid()) {

‎be/src/olap/rowset/rowset_meta_manager.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ Status RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_
8888
}
8989
Status RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id,
9090
const RowsetMetaPB& rowset_meta_pb, bool enable_binlog) {
91+
if (rowset_meta_pb.partition_id() <= 0) {
92+
LOG(WARNING) << "invalid partition id " << rowset_meta_pb.partition_id() << " tablet "
93+
<< rowset_meta_pb.tablet_id();
94+
return Status::InternalError("invalid partition id {}, tablet {}",
95+
rowset_meta_pb.partition_id(), rowset_meta_pb.tablet_id());
96+
}
9197
if (enable_binlog) {
9298
return _save_with_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb);
9399
} else {

‎be/src/olap/tablet_meta_manager.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,12 @@ Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash
9393
std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
9494
std::string value;
9595
static_cast<void>(tablet_meta->serialize(&value));
96+
if (tablet_meta->partition_id() <= 0) {
97+
LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << " tablet "
98+
<< tablet_meta->tablet_id();
99+
return Status::InternalError("invaid partition id {} tablet {}",
100+
tablet_meta->partition_id(), tablet_meta->tablet_id());
101+
}
96102
OlapMeta* meta = store->get_meta();
97103
VLOG_NOTICE << "save tablet meta"
98104
<< ", key:" << key << ", meta length:" << value.length();

‎be/src/olap/txn_manager.cpp

+8-11
Original file line numberDiff line numberDiff line change
@@ -295,9 +295,10 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id,
295295
const RowsetSharedPtr& rowset_ptr, PendingRowsetGuard guard,
296296
bool is_recovery) {
297297
if (partition_id < 1 || transaction_id < 1 || tablet_id < 1) {
298-
LOG(FATAL) << "invalid commit req "
299-
<< " partition_id=" << partition_id << " transaction_id=" << transaction_id
300-
<< " tablet_id=" << tablet_id;
298+
LOG(WARNING) << "invalid commit req "
299+
<< " partition_id=" << partition_id << " transaction_id=" << transaction_id
300+
<< " tablet_id=" << tablet_id;
301+
return Status::InternalError("invalid partition id");
301302
}
302303

303304
pair<int64_t, int64_t> key(partition_id, transaction_id);
@@ -383,10 +384,8 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id,
383384
}
384385
});
385386
if (!save_status.ok()) {
386-
return Status::Error<ROWSET_SAVE_FAILED>(
387-
"save committed rowset failed. when commit txn rowset_id: {}, tablet id: {}, "
388-
"txn id: {}",
389-
rowset_ptr->rowset_id().to_string(), tablet_id, transaction_id);
387+
save_status.append(fmt::format(", txn id: {}", transaction_id));
388+
return save_status;
390389
}
391390
}
392391

@@ -537,10 +536,8 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id,
537536
rowset->rowset_meta()->get_rowset_pb(), enable_binlog);
538537
stats->save_meta_time_us += MonotonicMicros() - t5;
539538
if (!status.ok()) {
540-
return Status::Error<ROWSET_SAVE_FAILED>(
541-
"save committed rowset failed. when publish txn rowset_id: {}, tablet id: {}, txn "
542-
"id: {}",
543-
rowset->rowset_id().to_string(), tablet_id, transaction_id);
539+
status.append(fmt::format(", txn id: {}", transaction_id));
540+
return status;
544541
}
545542

546543
// TODO(Drogon): remove these test codes

‎be/test/olap/delete_handler_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ static void tear_down() {
102102
static void set_default_create_tablet_request(TCreateTabletReq* request) {
103103
request->tablet_id = 10003;
104104
request->__set_version(1);
105+
request->partition_id = 10004;
105106
request->tablet_schema.schema_hash = 270068375;
106107
request->tablet_schema.short_key_column_count = 2;
107108
request->tablet_schema.keys_type = TKeysType::AGG_KEYS;

‎be/test/olap/delta_writer_test.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
105105
TCreateTabletReq* request) {
106106
request->tablet_id = tablet_id;
107107
request->__set_version(1);
108+
request->partition_id = 10001;
108109
request->tablet_schema.schema_hash = schema_hash;
109110
request->tablet_schema.short_key_column_count = 6;
110111
request->tablet_schema.keys_type = TKeysType::AGG_KEYS;
@@ -268,6 +269,7 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s
268269
bool enable_mow = false) {
269270
request->tablet_id = tablet_id;
270271
request->__set_version(1);
272+
request->partition_id = 30004;
271273
request->tablet_schema.schema_hash = schema_hash;
272274
request->tablet_schema.short_key_column_count = 2;
273275
request->tablet_schema.keys_type = TKeysType::UNIQUE_KEYS;

‎be/test/olap/engine_storage_migration_task_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s
102102
TCreateTabletReq* request) {
103103
request->tablet_id = tablet_id;
104104
request->__set_version(1);
105+
request->partition_id = 10001;
105106
request->tablet_schema.schema_hash = schema_hash;
106107
request->tablet_schema.short_key_column_count = 2;
107108
request->tablet_schema.keys_type = TKeysType::UNIQUE_KEYS;

‎be/test/olap/memtable_memory_limiter_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
3333
TCreateTabletReq* request) {
3434
request->tablet_id = tablet_id;
3535
request->__set_version(1);
36+
request->partition_id = 30002;
3637
request->tablet_schema.schema_hash = schema_hash;
3738
request->tablet_schema.short_key_column_count = 3;
3839
request->tablet_schema.keys_type = TKeysType::AGG_KEYS;

‎be/test/olap/ordered_data_compaction_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ class OrderedDataCompactionTest : public ::testing::Test {
271271
std::string json_rowset_meta = R"({
272272
"rowset_id": 540085,
273273
"tablet_id": 15674,
274+
"partition_id": 10000,
274275
"txn_id": 4045,
275276
"tablet_schema_hash": 567997588,
276277
"rowset_type": "BETA_ROWSET",

‎be/test/olap/path_gc_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ TEST(PathGcTest, GcTabletAndRowset) {
5454
auto create_tablet = [&](int64_t tablet_id) {
5555
auto tablet_meta = std::make_shared<TabletMeta>();
5656
tablet_meta->_tablet_id = tablet_id;
57+
(void)tablet_meta->set_partition_id(10000);
5758
tablet_meta->set_tablet_uid({tablet_id, 0});
5859
tablet_meta->set_shard_id(tablet_id % 4);
5960
tablet_meta->_schema_hash = tablet_id;

‎be/test/olap/remote_rowset_gc_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s
112112
TCreateTabletReq* request) {
113113
request->tablet_id = tablet_id;
114114
request->__set_version(1);
115+
request->partition_id = 30003;
115116
request->tablet_schema.schema_hash = schema_hash;
116117
request->tablet_schema.short_key_column_count = 2;
117118
request->tablet_schema.keys_type = TKeysType::UNIQUE_KEYS;

‎be/test/olap/rowid_conversion_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ class TestRowIdConversion : public testing::TestWithParam<std::tuple<KeysType, b
223223
std::string json_rowset_meta = R"({
224224
"rowset_id": 540081,
225225
"tablet_id": 15673,
226+
"partition_id": 10000,
226227
"tablet_schema_hash": 567997577,
227228
"rowset_type": "BETA_ROWSET",
228229
"rowset_state": "VISIBLE",

‎be/test/olap/segcompaction_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ class SegCompactionTest : public testing::Test {
194194
std::shared_ptr<DataDir> data_dir = std::make_shared<DataDir>(lTestDir);
195195
TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
196196
tablet_meta->_tablet_id = 1;
197+
tablet_meta->set_partition_id(10000);
197198
tablet_meta->_schema = tablet_schema;
198199
auto tablet = std::make_shared<Tablet>(tablet_meta, data_dir.get(), "test_str");
199200
char* tmp_str = (char*)malloc(20);

‎be/test/olap/tablet_cooldown_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ class TabletCooldownTest : public testing::Test {
267267
static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash,
268268
TCreateTabletReq* request) {
269269
request->tablet_id = tablet_id;
270+
request->partition_id = 30003;
270271
request->__set_version(1);
271272
request->tablet_schema.schema_hash = schema_hash;
272273
request->tablet_schema.short_key_column_count = 2;

‎be/test/olap/test_data/rowset_meta.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"rowset_id": 10000,
3+
"partition_id": 10001,
34
"tablet_id": 12046,
45
"tablet_schema_hash": 365187263,
56
"rowset_type": "BETA_ROWSET",

‎be/test/olap/test_data/rowset_meta2.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"rowset_id": 10001,
3+
"partition_id": 10001,
34
"tablet_id": 20487,
45
"tablet_schema_hash": 1520686811,
56
"rowset_type": "BETA_ROWSET",

‎be/test/runtime/load_stream_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ void construct_schema(OlapTableSchemaParam* schema) {
142142
static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
143143
TCreateTabletReq* request) {
144144
request->tablet_id = tablet_id;
145+
request->partition_id = 30001;
145146
request->__set_version(1);
146147
request->tablet_schema.schema_hash = schema_hash;
147148
request->tablet_schema.short_key_column_count = 6;

‎be/test/vec/olap/vertical_compaction_test.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ class VerticalCompactionTest : public ::testing::Test {
273273
std::string json_rowset_meta = R"({
274274
"rowset_id": 540081,
275275
"tablet_id": 15673,
276+
"partition_id": 10000,
276277
"tablet_schema_hash": 567997577,
277278
"rowset_type": "BETA_ROWSET",
278279
"rowset_state": "VISIBLE",

0 commit comments

Comments
 (0)