From d8b879ecd269527c481ce219c5870f276d06fc96 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Fri, 2 Dec 2022 16:58:52 +0100 Subject: [PATCH 1/5] Added support for ALTER TABLE t REORGANIZE PARTITION p INTO (..) --- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 1 + dbms/src/TiDB/Schema/SchemaGetter.h | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 290e39f4796..6a79fc4e35c 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -529,6 +529,7 @@ void SchemaBuilder::applyDiff(const SchemaDiff & diff) case SchemaActionType::AddTablePartition: case SchemaActionType::DropTablePartition: case SchemaActionType::TruncateTablePartition: + case SchemaActionType::AlterTableReorganizePartition: { applyPartitionDiff(db_info, diff.table_id); break; diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index b03ea01e5bb..9548e7f9ec3 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -95,11 +95,12 @@ enum class SchemaActionType : Int8 AlterNoCacheTable = 59, CreateTables = 60, ActionMultiSchemaChange = 61, + AlterTableReorganizePartition = 64, - // If we supporte new type from TiDB. + // If we support new type from TiDB. // MaxRecognizedType also needs to be changed. // It should always be equal to the maximum supported type + 1 - MaxRecognizedType = 62, + MaxRecognizedType = 65, }; struct AffectedOption From 0ad6630b1ae881161deeb554311483fd0424493e Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 7 Dec 2022 11:35:54 +0100 Subject: [PATCH 2/5] Add intermediate partitions to the table --- dbms/src/Storages/Transaction/TiDB.cpp | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 2f9790bcb37..790831c0ce5 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -525,10 +525,38 @@ try auto defs_json = json->getArray("definitions"); definitions.clear(); + std::unordered_set part_id_set; for (size_t i = 0; i < defs_json->size(); i++) { PartitionDefinition definition(defs_json->getObject(i)); definitions.emplace_back(definition); + part_id_set.emplace(definition.id); + } + + auto add_defs_json = json->getArray("adding_definitions"); + if (!add_defs_json.isNull()) { + for (size_t i = 0; i < add_defs_json->size(); i++) + { + PartitionDefinition definition(add_defs_json->getObject(i)); + if (part_id_set.count(definition.id) == 0) + { + definitions.emplace_back(definition); + part_id_set.emplace(definition.id); + } + } + } + + auto drop_defs_json = json->getArray("dropping_definitions"); + if (!drop_defs_json.isNull()) { + for (size_t i = 0; i < drop_defs_json->size(); i++) + { + PartitionDefinition definition(drop_defs_json->getObject(i)); + if (part_id_set.count(definition.id) == 0) + { + definitions.emplace_back(definition); + part_id_set.emplace(definition.id); + } + } } num = json->getValue("num"); From 2c7382ecb25a9ad915494c3db318f3f260f2cadc Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 7 Dec 2022 14:54:40 +0100 Subject: [PATCH 3/5] Formatting --- dbms/src/Storages/Transaction/TiDB.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 790831c0ce5..3f63f4053a6 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -534,7 +534,8 @@ try } auto add_defs_json = json->getArray("adding_definitions"); - if (!add_defs_json.isNull()) { + if (!add_defs_json.isNull()) + { for (size_t i = 0; i < add_defs_json->size(); i++) { PartitionDefinition definition(add_defs_json->getObject(i)); @@ -547,7 +548,8 @@ try } auto drop_defs_json = json->getArray("dropping_definitions"); - if (!drop_defs_json.isNull()) { + if (!drop_defs_json.isNull()) + { for (size_t i = 0; i < drop_defs_json->size(); i++) { PartitionDefinition definition(drop_defs_json->getObject(i)); From 28270f7b7f1c21b87d60b3ad2db4232888c047b0 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Dec 2022 10:22:12 +0100 Subject: [PATCH 4/5] Added missing TiDB ActionType's --- dbms/src/TiDB/Schema/SchemaGetter.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index 9548e7f9ec3..016b64557ab 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -95,12 +95,16 @@ enum class SchemaActionType : Int8 AlterNoCacheTable = 59, CreateTables = 60, ActionMultiSchemaChange = 61, + FlashbackCluster = 62, + RecoverSchema = 63, AlterTableReorganizePartition = 64, + AlterTTLInfo = 65, + AlterTTLRemove = 67, // If we support new type from TiDB. // MaxRecognizedType also needs to be changed. // It should always be equal to the maximum supported type + 1 - MaxRecognizedType = 65, + MaxRecognizedType = 68, }; struct AffectedOption From 7029d67c8295746542c9b7ac99651c526783f28a Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Thu, 9 Feb 2023 09:48:35 +0100 Subject: [PATCH 5/5] Added fullstack test for reorganize partition --- .../ddl/reorganize_partition.test | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/fullstack-test2/ddl/reorganize_partition.test diff --git a/tests/fullstack-test2/ddl/reorganize_partition.test b/tests/fullstack-test2/ddl/reorganize_partition.test new file mode 100644 index 00000000000..a0327e461f2 --- /dev/null +++ b/tests/fullstack-test2/ddl/reorganize_partition.test @@ -0,0 +1,101 @@ +# Copyright 2023 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +mysql> drop table if exists test.t; +mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000)); +mysql> analyze table test.t; +mysql> alter table test.t set tiflash replica 1; + +func> wait_table test t + +# check table info in tiflash +>> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 +┌─tidb_database─┬─tidb_name─┐ +│ test │ t │ +└───────────────┴───────────┘ + +mysql> insert into test.t values (1,"1",-1); +mysql> insert into test.t select a+1,a+1,-(a+1) from test.t; +mysql> insert into test.t select a+2,a+2,-(a+2) from test.t; +mysql> insert into test.t select a+500000,a+500000,-(a+500000) from test.t; +mysql> insert into test.t select a+1000000,a+1000000,-(a+1000000) from test.t; +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; + +mysql> alter table test.t reorganize partition p0 INTO (partition p0 values less than (500000), partition p500k values less than (1000000)); + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p500k); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p500k); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; +