diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 8383a777c0007..c0c7a69ca4b12 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -1221,9 +1221,15 @@ func (s *testIntegrationSuite5) TestAlterTableTruncatePartitionByList(c *C) { tk.MustExec(`insert into t values (1),(3),(5),(null)`) oldTbl := testGetTableByName(c, tk.Se, "test", "t") tk.MustExec(`alter table t truncate partition p1`) +<<<<<<< HEAD tk.MustQuery("select * from t").Check(testkit.Rows("1", "5", "")) tbl := testGetTableByName(c, tk.Se, "test", "t") c.Assert(tbl.Meta().Partition, NotNil) +======= + tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1", "5", "")) + tbl := tk.GetTableByName("test", "t") + require.NotNil(t, tbl.Meta().Partition) +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) part := tbl.Meta().Partition c.Assert(part.Type == model.PartitionTypeList, IsTrue) c.Assert(part.Definitions, HasLen, 3) @@ -1252,9 +1258,15 @@ func (s *testIntegrationSuite5) TestAlterTableTruncatePartitionByListColumns(c * tk.MustExec(`insert into t values (1,'a'),(3,'a'),(5,'a'),(null,null)`) oldTbl := testGetTableByName(c, tk.Se, "test", "t") tk.MustExec(`alter table t truncate partition p1`) +<<<<<<< HEAD tk.MustQuery("select * from t").Check(testkit.Rows("1 a", "5 a", " ")) tbl := testGetTableByName(c, tk.Se, "test", "t") c.Assert(tbl.Meta().Partition, NotNil) +======= + tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1 a", "5 a", " ")) + tbl := tk.GetTableByName("test", "t") + require.NotNil(t, tbl.Meta().Partition) +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) part := tbl.Meta().Partition c.Assert(part.Type == model.PartitionTypeList, IsTrue) c.Assert(part.Definitions, HasLen, 3) diff --git a/executor/builder.go b/executor/builder.go index ec1affd39b346..cf40c0ab492b2 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3292,14 +3292,21 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() +<<<<<<< HEAD if v.IsCommonHandle { if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc) +======= + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if v.IsCommonHandle { + kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) if err != nil { return nil, err } return builder.buildTableReaderFromKvRanges(ctx, e, kvRanges) } +<<<<<<< HEAD tbl, _ := builder.is.TableByID(tbInfo.ID) pt := tbl.(table.PartitionedTable) @@ -3312,6 +3319,31 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte var kvRanges []kv.KeyRange if keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { // In this case we can use dynamic partition pruning. +======= + handles, _ := dedupHandles(lookUpContents) + return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) + } + tbl, _ := builder.is.TableByID(tbInfo.ID) + pt := tbl.(table.PartitionedTable) + pe, err := tbl.(interface { + PartitionExpr() (*tables.PartitionExpr, error) + }).PartitionExpr() + if err != nil { + return nil, err + } + partitionInfo := &v.PartitionInfo + usedPartitionList, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) + if err != nil { + return nil, err + } + usedPartitions := make(map[int64]table.PhysicalTable, len(usedPartitionList)) + for _, p := range usedPartitionList { + usedPartitions[p.GetPhysicalID()] = p + } + var kvRanges []kv.KeyRange + if v.IsCommonHandle { + if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) for _, content := range lookUpContents { @@ -3323,13 +3355,21 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } pid := p.GetPhysicalID() +<<<<<<< HEAD tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, []*indexJoinLookUpContent{content}, indexRanges, keyOff2IdxOff, cwc) +======= + if _, ok := usedPartitions[pid]; !ok { + continue + } + tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, []*indexJoinLookUpContent{content}, indexRanges, keyOff2IdxOff, cwc, nil, interruptSignal) +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) if err != nil { return nil, err } kvRanges = append(kvRanges, tmp...) } } else { +<<<<<<< HEAD partitionInfo := &v.PartitionInfo partitions, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) if err != nil { @@ -3339,6 +3379,11 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte for _, p := range partitions { pid := p.GetPhysicalID() tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc) +======= + kvRanges = make([]kv.KeyRange, 0, len(usedPartitions)*len(lookUpContents)) + for _, p := range usedPartitionList { + tmp, err := buildKvRangesForIndexJoin(e.ctx, p.GetPhysicalID(), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) if err != nil { return nil, err } @@ -3349,13 +3394,8 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte } handles, lookUpContents := dedupHandles(lookUpContents) - if tbInfo.GetPartitionInfo() == nil { - return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) - } - if !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { - return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) - } +<<<<<<< HEAD tbl, _ := builder.is.TableByID(tbInfo.ID) pt := tbl.(table.PartitionedTable) pe, err := tbl.(interface { @@ -3366,6 +3406,9 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte } var kvRanges []kv.KeyRange if keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { +======= + if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) for _, content := range lookUpContents { @@ -3377,19 +3420,16 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } pid := p.GetPhysicalID() + if _, ok := usedPartitions[pid]; !ok { + continue + } handle := kv.IntHandle(content.keys[0].GetInt64()) tmp := distsql.TableHandlesToKVRanges(pid, []kv.Handle{handle}) kvRanges = append(kvRanges, tmp...) } } else { - partitionInfo := &v.PartitionInfo - partitions, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) - if err != nil { - return nil, err - } - for _, p := range partitions { - pid := p.GetPhysicalID() - tmp := distsql.TableHandlesToKVRanges(pid, handles) + for _, p := range usedPartitionList { + tmp := distsql.TableHandlesToKVRanges(p.GetPhysicalID(), handles) kvRanges = append(kvRanges, tmp...) } } diff --git a/executor/distsql.go b/executor/distsql.go index a0519dd460622..caee1e34e1623 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -319,6 +319,16 @@ type IndexLookUpExecutor struct { // columns are only required by union scan. columns []*model.ColumnInfo *dataReaderBuilder +<<<<<<< HEAD +======= + + // fields about accessing partition tables + partitionTableMode bool // if this executor is accessing a partition table + prunedPartitions []table.PhysicalTable // partition tables need to access + partitionRangeMap map[int64][]*ranger.Range + partitionKVRanges [][]kv.KeyRange // kvRanges of each prunedPartitions + +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093) // All fields above are immutable. idxWorkerWg sync.WaitGroup diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 56830615128c1..66b38c1276635 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -978,6 +978,7 @@ func (p *LogicalJoin) constructInnerTableScanTask( Columns: ds.TblCols, ColumnNames: ds.names, } + ts.PartitionInfo = copTask.partitionInfo selStats := ts.stats.Scale(selectivity) ts.addPushedDownSelection(copTask, selStats) t := copTask.convertToRootTask(ds.ctx) diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index 9b7898e23b891..f9c805fd401a0 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -394,7 +394,7 @@ func (s *testPartitionPruneSuit) TestIssue22898(c *C) { tk.MustExec("CREATE TABLE NT_RP3763 (COL1 TINYINT(8) SIGNED COMMENT \"NUMERIC NO INDEX\" DEFAULT 41,COL2 VARCHAR(20),COL3 DATETIME,COL4 BIGINT,COL5 FLOAT) PARTITION BY RANGE (COL1 * COL3) (PARTITION P0 VALUES LESS THAN (0),PARTITION P1 VALUES LESS THAN (10),PARTITION P2 VALUES LESS THAN (20),PARTITION P3 VALUES LESS THAN (30),PARTITION P4 VALUES LESS THAN (40),PARTITION P5 VALUES LESS THAN (50),PARTITION PMX VALUES LESS THAN MAXVALUE);") tk.MustExec("insert into NT_RP3763 (COL1,COL2,COL3,COL4,COL5) values(-82,\"夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇\",\"5748\\-06\\-26\\ 20:48:49\",-3133527360541070260,-2.624880003397658e+38);") tk.MustExec("insert into NT_RP3763 (COL1,COL2,COL3,COL4,COL5) values(48,\"簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢\",\"7228\\-12\\-13\\ 02:59:54\",-6181009269190017937,2.7731105531290494e+38);") - tk.MustQuery("select * from `NT_RP3763` where `COL1` in (10, 48, -82);").Check(testkit.Rows("-82 夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇 5748-06-26 20:48:49 -3133527360541070260 -262488000000000000000000000000000000000", "48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) + tk.MustQuery("select * from `NT_RP3763` where `COL1` in (10, 48, -82);").Sort().Check(testkit.Rows("-82 夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇 5748-06-26 20:48:49 -3133527360541070260 -262488000000000000000000000000000000000", "48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) tk.MustQuery("select * from `NT_RP3763` where `COL1` in (48);").Check(testkit.Rows("48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) } @@ -463,3 +463,83 @@ partition by range (a) ( tk.MustQuery("select * from t3 where not (a = 1)").Sort().Check(testkit.Rows("11", "12", "13", "2", "3")) tk.MustQuery("select * from t3 where not (a != 1)").Check(testkit.Rows("1")) } +<<<<<<< HEAD +======= + +//issue 22079 +func TestRangePartitionPredicatePruner(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@tidb_partition_prune_mode='" + string(variable.Static) + "'") + tk.MustExec("drop database if exists test_partition;") + tk.MustExec("create database test_partition") + tk.MustExec("use test_partition") + tk.MustExec("drop table if exists t") + tk.Session().GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly + tk.MustExec(`create table t (a int(11) default null) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin + partition by range(a) ( + partition p0 values less than (1), + partition p1 values less than (2), + partition p2 values less than (3), + partition p_max values less than (maxvalue));`) + + var input []string + var output []struct { + SQL string + Result []string + } + partitionPrunerData := plannercore.GetPartitionPrunerData() + partitionPrunerData.GetTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Result = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + tk.MustQuery(tt).Check(testkit.Rows(output[i].Result...)) + } +} + +func TestHashPartitionPruning(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@tidb_partition_prune_mode='static'") + tk.MustExec("USE test;") + tk.MustExec("DROP TABLE IF EXISTS t;") + tk.MustExec("CREATE TABLE t (`COL1` int, `COL3` bigint) PARTITION BY HASH ((`COL1` * `COL3`))PARTITIONS 13;") + tk.MustQuery("SELECT * FROM t WHERE col3 =2659937067964964513 and col1 = 783367513002;").Check(testkit.Rows()) + tk.MustExec("drop table if exists t;") + tk.MustExec("CREATE TABLE `t` (" + + "`COL1` int NOT NULL DEFAULT '25' COMMENT 'NUMERIC PK'," + + "`COL3` bigint NOT NULL," + + "PRIMARY KEY (`COL1`,`COL3`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin " + + "PARTITION BY HASH ((`COL1` * `COL3`))" + + "PARTITIONS 13;") + tk.MustExec("insert into t(col1, col3) values(0, 3522101843073676459);") + tk.MustQuery("SELECT col1, COL3 FROM t WHERE COL1 IN (0,14158354938390,0) AND COL3 IN (3522101843073676459,-2846203247576845955,838395691793635638);").Check(testkit.Rows("0 3522101843073676459")) +} + +func TestIssue32007(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("create database Issue32007") + tk.MustExec("USE Issue32007") + tk.MustExec("create table t1 (a int, b tinyint, primary key (a)) partition by range (a) (" + + "partition p0 values less than (5)," + + "partition p1 values less than (20)," + + "partition p2 values less than (30)," + + "partition p3 values less than (40)," + + "partition p4 values less than MAXVALUE)") + tk.MustExec("insert into t1 values (0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (20, 20), (21, 21), (22, 22), (23, 23), (24, 24), (25, 25), (30, 30), (31, 31), (32, 32), (33, 33), (34, 34), (35, 35), (36, 36), (40, 40), (50, 50), (80, 80), (90, 90), (100, 100)") + tk.MustExec("create table t3 (a int, b mediumint, primary key (a))") + tk.MustExec("insert into t3 values (0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17), (18, 18), (19, 19), (20, 20), (21, 21), (22, 22), (23, 23)") + + tk.MustExec("set @@tidb_partition_prune_mode='static'") + tk.MustQuery("select * from t3 where t3.a <> ALL (select t1.a from t1 partition (p0)) order by t3.a").Sort().Check(testkit.Rows("10 10", "11 11", "12 12", "13 13", "14 14", "15 15", "16 16", "17 17", "18 18", "19 19", "20 20", "21 21", "22 22", "23 23", "5 5", "6 6", "7 7", "8 8", "9 9")) + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + tk.MustQuery("select * from t3 where t3.a <> ALL (select t1.a from t1 partition (p0)) order by t3.a").Sort().Check(testkit.Rows("10 10", "11 11", "12 12", "13 13", "14 14", "15 15", "16 16", "17 17", "18 18", "19 19", "20 20", "21 21", "22 22", "23 23", "5 5", "6 6", "7 7", "8 8", "9 9")) +} +>>>>>>> b9bd5a7d7... *: add explicit partition pruning to index joins (#32007) (#32093)