diff --git a/planner/core/integration_partition_test.go b/planner/core/integration_partition_test.go index 9df4c962541ee..b17dcf6c72812 100644 --- a/planner/core/integration_partition_test.go +++ b/planner/core/integration_partition_test.go @@ -1655,3 +1655,43 @@ func TestPartitionProcessorWithUninitializedTable(t *testing.T) { } tk.MustQuery("explain format=brief select * from q1,q2").CheckAt([]int{0}, rows) } + +func TestEstimationForTopNPushToDynamicPartition(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set tidb_cost_model_version=2") + tk.MustExec("drop table if exists tlist") + tk.MustExec(`set tidb_enable_list_partition = 1`) + tk.MustExec(`create table trange (a int, b int, c int, index ia(a), primary key (b) clustered) + partition by range(b) ( + partition p1 values less than(100), + partition p2 values less than(200), + partition p3 values less than maxvalue);`) + tk.MustExec(`create table tlist (a int, b int, c int, index ia(a), primary key (b) clustered) + partition by list (b) ( + partition p0 values in (0, 1, 2), + partition p1 values in (3, 4, 5));`) + tk.MustExec(`create table thash (a int, b int, c int, index ia(a), primary key (b) clustered) + partition by hash(b) partitions 4;`) + tk.MustExec(`create table t (a int, b int, c int, index ia(a), primary key (b) clustered);`) + tk.MustExec(`analyze table trange;`) + tk.MustExec(`analyze table tlist;`) + tk.MustExec(`analyze table thash;`) + tk.MustExec(`analyze table t;`) + + var input []string + var output []struct { + SQL string + Plan []string + } + integrationPartitionSuiteData := core.GetIntegrationPartitionSuiteData() + integrationPartitionSuiteData.LoadTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) + } +} diff --git a/planner/core/plan.go b/planner/core/plan.go index e492ad08d0fc7..cdd2b550cac87 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -368,6 +368,9 @@ type PhysicalPlan interface { // Stats returns the StatsInfo of the plan. Stats() *property.StatsInfo + // SetStats sets basePlan.stats inside the basePhysicalPlan. + SetStats(s *property.StatsInfo) + // ExplainNormalizedInfo returns operator normalized information for generating digest. ExplainNormalizedInfo() string @@ -815,6 +818,11 @@ func (p *basePlan) Stats() *property.StatsInfo { return p.stats } +// SetStats sets basePlan.stats +func (p *basePlan) SetStats(s *property.StatsInfo) { + p.stats = s +} + // basePlanSize is the size of basePlan. const basePlanSize = int64(unsafe.Sizeof(basePlan{})) diff --git a/planner/core/task.go b/planner/core/task.go index c0fc7a9bfaad7..aa566e0dcc0cd 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1032,6 +1032,10 @@ func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bo return true } var ( + selOnIdxScan *PhysicalSelection + selOnTblScan *PhysicalSelection + selSelectivity float64 + idxScan *PhysicalIndexScan tblScan *PhysicalTableScan tblInfo *model.TableInfo @@ -1044,6 +1048,7 @@ func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bo } finalIdxScanPlan := copTsk.indexPlan for len(finalIdxScanPlan.Children()) > 0 && finalIdxScanPlan.Children()[0] != nil { + selOnIdxScan, _ = finalIdxScanPlan.(*PhysicalSelection) finalIdxScanPlan = finalIdxScanPlan.Children()[0] } idxScan = finalIdxScanPlan.(*PhysicalIndexScan) @@ -1056,12 +1061,21 @@ func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bo } finalTblScanPlan := copTsk.tablePlan for len(finalTblScanPlan.Children()) > 0 { + selOnTblScan, _ = finalTblScanPlan.(*PhysicalSelection) finalTblScanPlan = finalTblScanPlan.Children()[0] } tblScan = finalTblScanPlan.(*PhysicalTableScan) tblInfo = tblScan.Table } + // Note that we only need to care about one Selection at most. + if selOnIdxScan != nil && idxScan.statsInfo().RowCount > 0 { + selSelectivity = selOnIdxScan.statsInfo().RowCount / idxScan.statsInfo().RowCount + } + if idxScan == nil && selOnTblScan != nil && tblScan.statsInfo().RowCount > 0 { + selSelectivity = selOnTblScan.statsInfo().RowCount / tblScan.statsInfo().RowCount + } + pi := tblInfo.GetPartitionInfo() if pi == nil { return nil, false @@ -1083,6 +1097,17 @@ func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bo }.Init(p.SCtx(), stats, p.SelectBlockOffset()) pushedLimit.SetSchema(copTsk.indexPlan.Schema()) copTsk = attachPlan2Task(pushedLimit, copTsk).(*copTask) + + // A similar but simplified logic compared the ExpectedCnt handling logic in getOriginalPhysicalIndexScan. + child := pushedLimit.Children()[0] + // The row count of the direct child of Limit should be adjusted to be no larger than the Limit.Count. + child.SetStats(child.statsInfo().ScaleByExpectCnt(float64(newCount))) + // The Limit->Selection->IndexScan case: + // adjust the row count of IndexScan according to the selectivity of the Selection. + if selSelectivity > 0 && selSelectivity < 1 { + scaledRowCount := child.Stats().RowCount / selSelectivity + idxScan.SetStats(idxScan.Stats().ScaleByExpectCnt(scaledRowCount)) + } } else if copTsk.indexPlan == nil { if tblScan.HandleCols == nil { return nil, false @@ -1111,6 +1136,17 @@ func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bo }.Init(p.SCtx(), stats, p.SelectBlockOffset()) pushedLimit.SetSchema(copTsk.tablePlan.Schema()) copTsk = attachPlan2Task(pushedLimit, copTsk).(*copTask) + + // A similar but simplified logic compared the ExpectedCnt handling logic in getOriginalPhysicalTableScan. + child := pushedLimit.Children()[0] + // The row count of the direct child of Limit should be adjusted to be no larger than the Limit.Count. + child.SetStats(child.statsInfo().ScaleByExpectCnt(float64(newCount))) + // The Limit->Selection->TableScan case: + // adjust the row count of IndexScan according to the selectivity of the Selection. + if selSelectivity > 0 && selSelectivity < 1 { + scaledRowCount := child.Stats().RowCount / selSelectivity + tblScan.SetStats(tblScan.Stats().ScaleByExpectCnt(scaledRowCount)) + } } else { return nil, false } diff --git a/planner/core/testdata/integration_partition_suite_in.json b/planner/core/testdata/integration_partition_suite_in.json index a31a8531437d8..6a8f2fc0af486 100644 --- a/planner/core/testdata/integration_partition_suite_in.json +++ b/planner/core/testdata/integration_partition_suite_in.json @@ -149,5 +149,34 @@ "explain format='brief' select a from tcollist limit 10", "explain format='brief' select a from tcollist order by a limit 10" ] + }, + { + "name": "TestEstimationForTopNPushToDynamicPartition", + "cases": [ + "explain format='brief' select a from t use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select a from trange use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select a from tlist use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select a from thash use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select * from t use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select * from trange use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select * from tlist use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select * from thash use index (ia) where a > 10 order by a limit 10", + "explain format='brief' select * from t use index (ia) where a + 1 > 10 order by a limit 10", + "explain format='brief' select * from trange use index (ia) where a + 1 > 10 order by a limit 10", + "explain format='brief' select * from tlist use index (ia) where a + 1 > 10 order by a limit 10", + "explain format='brief' select * from thash use index (ia) where a + 1 > 10 order by a limit 10", + "explain format='brief' select a from t use index (ia) where a > 10 and c = 10 order by a limit 10", + "explain format='brief' select a from trange use index (ia) where a > 10 and c = 10 order by a limit 10", + "explain format='brief' select a from tlist use index (ia) where a > 10 and c = 10 order by a limit 10", + "explain format='brief' select a from thash use index (ia) where a > 10 and c = 10 order by a limit 10", + "explain format='brief' select a from t use index () where b > 10 order by b limit 10", + "explain format='brief' select a from trange use index () where b > 10 order by b limit 10", + "explain format='brief' select a from tlist use index () where b > 10 order by b limit 10", + "explain format='brief' select a from thash use index () where b > 10 order by b limit 10", + "explain format='brief' select a from t use index () where a > 10 order by b limit 10", + "explain format='brief' select a from trange use index () where a > 10 order by b limit 10", + "explain format='brief' select a from tlist use index () where a > 10 order by b limit 10", + "explain format='brief' select a from thash use index () where a > 10 order by b limit 10" + ] } ] diff --git a/planner/core/testdata/integration_partition_suite_out.json b/planner/core/testdata/integration_partition_suite_out.json index e496996969211..9e9999cd9d4ba 100644 --- a/planner/core/testdata/integration_partition_suite_out.json +++ b/planner/core/testdata/integration_partition_suite_out.json @@ -1157,5 +1157,260 @@ ] } ] + }, + { + "Name": "TestEstimationForTopNPushToDynamicPartition", + "Cases": [ + { + "SQL": "explain format='brief' select a from t use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "Limit 10.00 root offset:0, count:10", + "└─IndexReader 10.00 root index:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─IndexRangeScan 10.00 cop[tikv] table:t, index:ia(a) range:(10,+inf], keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from trange use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.trange.a, offset:0, count:10", + "└─IndexReader 10.00 root partition:all index:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─IndexRangeScan 10.00 cop[tikv] table:trange, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from tlist use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.tlist.a, offset:0, count:10", + "└─IndexReader 10.00 root partition:all index:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─IndexRangeScan 10.00 cop[tikv] table:tlist, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from thash use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.thash.a, offset:0, count:10", + "└─IndexReader 10.00 root partition:all index:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─IndexRangeScan 10.00 cop[tikv] table:thash, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from t use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "IndexLookUp 10.00 root limit embedded(offset:0, count:10)", + "├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + "│ └─IndexRangeScan 10.00 cop[tikv] table:t, index:ia(a) range:(10,+inf], keep order:true, stats:pseudo", + "└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from trange use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.trange.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─IndexRangeScan 10.00 cop[tikv] table:trange, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:trange keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from tlist use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.tlist.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─IndexRangeScan 10.00 cop[tikv] table:tlist, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:tlist keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from thash use index (ia) where a > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.thash.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─IndexRangeScan 10.00 cop[tikv] table:thash, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:thash keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from t use index (ia) where a + 1 > 10 order by a limit 10", + "Plan": [ + "IndexLookUp 10.00 root limit embedded(offset:0, count:10)", + "├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + "│ └─Selection 10.00 cop[tikv] gt(plus(test.t.a, 1), 10)", + "│ └─IndexFullScan 12.50 cop[tikv] table:t, index:ia(a) keep order:true, stats:pseudo", + "└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from trange use index (ia) where a + 1 > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.trange.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─Selection 10.00 cop[tikv] gt(plus(test.trange.a, 1), 10)", + " │ └─IndexFullScan 12.50 cop[tikv] table:trange, index:ia(a) keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:trange keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from tlist use index (ia) where a + 1 > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.tlist.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─Selection 10.00 cop[tikv] gt(plus(test.tlist.a, 1), 10)", + " │ └─IndexFullScan 12.50 cop[tikv] table:tlist, index:ia(a) keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:tlist keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from thash use index (ia) where a + 1 > 10 order by a limit 10", + "Plan": [ + "TopN 10.00 root test.thash.a, offset:0, count:10", + "└─IndexLookUp 10.00 root partition:all ", + " ├─Limit(Build) 10.00 cop[tikv] offset:0, count:10", + " │ └─Selection 10.00 cop[tikv] gt(plus(test.thash.a, 1), 10)", + " │ └─IndexFullScan 12.50 cop[tikv] table:thash, index:ia(a) keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:thash keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from t use index (ia) where a > 10 and c = 10 order by a limit 10", + "Plan": [ + "Projection 3.33 root test.t.a", + "└─Limit 3.33 root offset:0, count:10", + " └─Projection 3.33 root test.t.a, test.t.c", + " └─IndexLookUp 3.33 root ", + " ├─IndexRangeScan(Build) 3333.33 cop[tikv] table:t, index:ia(a) range:(10,+inf], keep order:true, stats:pseudo", + " └─Selection(Probe) 3.33 cop[tikv] eq(test.t.c, 10)", + " └─TableRowIDScan 3333.33 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from trange use index (ia) where a > 10 and c = 10 order by a limit 10", + "Plan": [ + "Projection 3.33 root test.trange.a", + "└─TopN 3.33 root test.trange.a, offset:0, count:10", + " └─IndexLookUp 3.33 root partition:all ", + " ├─IndexRangeScan(Build) 3333.33 cop[tikv] table:trange, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TopN(Probe) 3.33 cop[tikv] test.trange.a, offset:0, count:10", + " └─Selection 3.33 cop[tikv] eq(test.trange.c, 10)", + " └─TableRowIDScan 3333.33 cop[tikv] table:trange keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from tlist use index (ia) where a > 10 and c = 10 order by a limit 10", + "Plan": [ + "Projection 3.33 root test.tlist.a", + "└─TopN 3.33 root test.tlist.a, offset:0, count:10", + " └─IndexLookUp 3.33 root partition:all ", + " ├─IndexRangeScan(Build) 3333.33 cop[tikv] table:tlist, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TopN(Probe) 3.33 cop[tikv] test.tlist.a, offset:0, count:10", + " └─Selection 3.33 cop[tikv] eq(test.tlist.c, 10)", + " └─TableRowIDScan 3333.33 cop[tikv] table:tlist keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from thash use index (ia) where a > 10 and c = 10 order by a limit 10", + "Plan": [ + "Projection 3.33 root test.thash.a", + "└─TopN 3.33 root test.thash.a, offset:0, count:10", + " └─IndexLookUp 3.33 root partition:all ", + " ├─IndexRangeScan(Build) 3333.33 cop[tikv] table:thash, index:ia(a) range:(10,+inf], keep order:false, stats:pseudo", + " └─TopN(Probe) 3.33 cop[tikv] test.thash.a, offset:0, count:10", + " └─Selection 3.33 cop[tikv] eq(test.thash.c, 10)", + " └─TableRowIDScan 3333.33 cop[tikv] table:thash keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from t use index () where b > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.t.a", + "└─Limit 10.00 root offset:0, count:10", + " └─TableReader 10.00 root data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─TableRangeScan 10.00 cop[tikv] table:t range:(10,+inf], keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from trange use index () where b > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.trange.a", + "└─TopN 10.00 root test.trange.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─TableRangeScan 10.00 cop[tikv] table:trange range:(10,+inf], keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from tlist use index () where b > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.tlist.a", + "└─TopN 10.00 root test.tlist.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─TableRangeScan 10.00 cop[tikv] table:tlist range:(10,+inf], keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from thash use index () where b > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.thash.a", + "└─TopN 10.00 root test.thash.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─TableRangeScan 10.00 cop[tikv] table:thash range:(10,+inf], keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from t use index () where a > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.t.a", + "└─Limit 10.00 root offset:0, count:10", + " └─TableReader 10.00 root data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─Selection 10.00 cop[tikv] gt(test.t.a, 10)", + " └─TableFullScan 30.00 cop[tikv] table:t keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from trange use index () where a > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.trange.a", + "└─TopN 10.00 root test.trange.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─Selection 10.00 cop[tikv] gt(test.trange.a, 10)", + " └─TableFullScan 30.00 cop[tikv] table:trange keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from tlist use index () where a > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.tlist.a", + "└─TopN 10.00 root test.tlist.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─Selection 10.00 cop[tikv] gt(test.tlist.a, 10)", + " └─TableFullScan 30.00 cop[tikv] table:tlist keep order:true, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select a from thash use index () where a > 10 order by b limit 10", + "Plan": [ + "Projection 10.00 root test.thash.a", + "└─TopN 10.00 root test.thash.b, offset:0, count:10", + " └─TableReader 10.00 root partition:all data:Limit", + " └─Limit 10.00 cop[tikv] offset:0, count:10", + " └─Selection 10.00 cop[tikv] gt(test.thash.a, 10)", + " └─TableFullScan 30.00 cop[tikv] table:thash keep order:true, stats:pseudo" + ] + } + ] } ]