From 0f90724df23ff870a81b0a67c045c0eb8ff33324 Mon Sep 17 00:00:00 2001 From: xufei Date: Thu, 2 Dec 2021 14:23:34 +0800 Subject: [PATCH 1/8] show correct access object while explain mpp query over partition table --- planner/core/explain.go | 41 +++++++++++++++++++++++++++++++--- planner/core/physical_plans.go | 7 ++++++ planner/core/task.go | 13 +++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/planner/core/explain.go b/planner/core/explain.go index eda39f413a10e..72cd69ccccbd7 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -332,8 +332,7 @@ func (p *PhysicalTableReader) ExplainNormalizedInfo() string { return "" } -func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { - ts := p.TablePlans[0].(*PhysicalTableScan) +func getAccessObjectForTableScan(sctx sessionctx.Context, ts *PhysicalTableScan, partitionInfo PartitionInfo) string { pi := ts.Table.GetPartitionInfo() if pi == nil || !sctx.GetSessionVars().UseDynamicPartitionPrune() { return "" @@ -346,7 +345,43 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { } tbl := tmp.(table.PartitionedTable) - return partitionAccessObject(sctx, tbl, pi, &p.PartitionInfo) + return partitionAccessObject(sctx, tbl, pi, &partitionInfo) +} + +func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { + if !sctx.GetSessionVars().UseDynamicPartitionPrune() { + return "" + } + if len(p.PartitionInfos) == 0 { + ts := p.TablePlans[0].(*PhysicalTableScan) + return getAccessObjectForTableScan(sctx, ts, p.PartitionInfo) + } else { + if len(p.PartitionInfos) == 1 { + return getAccessObjectForTableScan(sctx, p.PartitionInfos[0].TableScan, p.PartitionInfos[0].PartitionInfo) + } + var buffer bytes.Buffer + for index, info := range p.PartitionInfos { + if index > 0 { + buffer.WriteString(", ") + } + + tblName := info.TableScan.Table.Name.O + if info.TableScan.TableAsName != nil && info.TableScan.TableAsName.O != "" { + tblName = info.TableScan.TableAsName.O + } + + if info.TableScan.Table.GetPartitionInfo() == nil { + buffer.WriteString("table of ") + buffer.WriteString(tblName) + continue + } else { + buffer.WriteString(getAccessObjectForTableScan(sctx, info.TableScan, info.PartitionInfo)) + buffer.WriteString(" of ") + buffer.WriteString(tblName) + } + } + return buffer.String() + } } func partitionAccessObject(sctx sessionctx.Context, tbl table.PartitionedTable, pi *model.PartitionInfo, partTable *PartitionInfo) string { diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index a4d8945121519..438b76522f3ce 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -68,6 +68,11 @@ var ( _ PhysicalPlan = &PhysicalTableSample{} ) +type TableScanAndPartitionInfo struct { + TableScan *PhysicalTableScan + PartitionInfo PartitionInfo +} + // PhysicalTableReader is the table reader in tidb. type PhysicalTableReader struct { physicalSchemaProducer @@ -86,6 +91,8 @@ type PhysicalTableReader struct { // Used by partition table. PartitionInfo PartitionInfo + // Used by MPP, because MPP plan may contain union, it is possible that a physical table reader contains more than 1 table scan + PartitionInfos []TableScanAndPartitionInfo } // PartitionInfo indicates partition helper info in physical plan. diff --git a/planner/core/task.go b/planner/core/task.go index 3133d9bcb9ac0..9151af6ac7555 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -2254,6 +2254,18 @@ func (t *mppTask) convertToRootTask(ctx sessionctx.Context) *rootTask { return t.copy().(*mppTask).convertToRootTaskImpl(ctx) } +func collectPartitionInfosFromMPPPlan(p *PhysicalTableReader, mppPlan PhysicalPlan) { + switch x := mppPlan.(type) { + case *PhysicalTableScan: + p.PartitionInfos = append(p.PartitionInfos, TableScanAndPartitionInfo{x, x.PartitionInfo}) + case *PhysicalExchangeReceiver: + default: + for _, ch := range mppPlan.Children() { + collectPartitionInfosFromMPPPlan(p, ch) + } + } +} + func (t *mppTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { sender := PhysicalExchangeSender{ ExchangeType: tipb.ExchangeType_PassThrough, @@ -2266,6 +2278,7 @@ func (t *mppTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { StoreType: kv.TiFlash, }.Init(ctx, t.p.SelectBlockOffset()) p.stats = t.p.statsInfo() + collectPartitionInfosFromMPPPlan(p, t.p) cst := t.cst + t.count()*ctx.GetSessionVars().GetNetworkFactor(nil) p.cost = cst / p.ctx.GetSessionVars().CopTiFlashConcurrencyFactor From 654a854a563d3b1aba25fb240c83d427ecc33222 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 09:39:40 +0800 Subject: [PATCH 2/8] add tests --- executor/partition_table_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index e742e3a057e79..71203e2f1430e 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -21,6 +21,7 @@ import ( "time" . "github.com/pingcap/check" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx/variable" @@ -1726,6 +1727,36 @@ func (s *partitionTableSuite) TestAddDropPartitions(c *C) { tk.MustPartition(`select * from t where a < 20`, "p1,p2,p3").Sort().Check(testkit.Rows("12", "15", "7")) } +func (s *partitionTableSuite) TestMPPQueryExplainInfo(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database tiflash_partition_test") + tk.MustExec("use tiflash_partition_test") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table t(a int) partition by range(a) ( + partition p0 values less than (5), + partition p1 values less than (10), + partition p2 values less than (15))`) + tb := testGetTableByName(c, tk.Se, "tiflash_partition_test", "t") + for _, partition := range tb.Meta().GetPartitionInfo().Definitions { + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, partition.ID, true) + c.Assert(err == nil, IsTrue) + } + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err == nil, IsTrue) + tk.MustExec(`insert into t values (2), (7), (12)`) + tk.MustExec("set tidb_enforce_mpp=1") + tk.MustPartition(`select * from t where a < 3`, "p0").Sort().Check(testkit.Rows("2")) + tk.MustPartition(`select * from t where a < 8`, "p0,p1").Sort().Check(testkit.Rows("2", "7")) + tk.MustPartition(`select * from t where a < 20`, "all").Sort().Check(testkit.Rows("12", "2", "7")) + tk.MustPartition(`select * from t where a < 5 union all select * from t where a > 10`, "p0").Sort().Check(testkit.Rows("12", "2")) + tk.MustPartition(`select * from t where a < 5 union all select * from t where a > 10`, "p2").Sort().Check(testkit.Rows("12", "2")) +} + func (s *partitionTableSuite) PartitionPruningInTransaction(c *C) { if israce.RaceEnabled { c.Skip("exhaustive types test, skip race test") From df2a30999085519fece9f72c1e1587032f944234 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 10:05:43 +0800 Subject: [PATCH 3/8] fix bug --- planner/core/task.go | 1 - 1 file changed, 1 deletion(-) diff --git a/planner/core/task.go b/planner/core/task.go index 9151af6ac7555..b5d595f6f28e6 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -2258,7 +2258,6 @@ func collectPartitionInfosFromMPPPlan(p *PhysicalTableReader, mppPlan PhysicalPl switch x := mppPlan.(type) { case *PhysicalTableScan: p.PartitionInfos = append(p.PartitionInfos, TableScanAndPartitionInfo{x, x.PartitionInfo}) - case *PhysicalExchangeReceiver: default: for _, ch := range mppPlan.Children() { collectPartitionInfosFromMPPPlan(p, ch) From b8f3e0ba308e4461d16306d42d7f20928dd216a9 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 10:27:30 +0800 Subject: [PATCH 4/8] fmt code --- planner/core/explain.go | 42 ++++++++++++++++------------------ planner/core/physical_plans.go | 8 +++---- planner/core/task.go | 2 +- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/planner/core/explain.go b/planner/core/explain.go index 72cd69ccccbd7..380f1b4dfd644 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -355,33 +355,31 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { if len(p.PartitionInfos) == 0 { ts := p.TablePlans[0].(*PhysicalTableScan) return getAccessObjectForTableScan(sctx, ts, p.PartitionInfo) - } else { - if len(p.PartitionInfos) == 1 { - return getAccessObjectForTableScan(sctx, p.PartitionInfos[0].TableScan, p.PartitionInfos[0].PartitionInfo) + } + if len(p.PartitionInfos) == 1 { + return getAccessObjectForTableScan(sctx, p.PartitionInfos[0].tableScan, p.PartitionInfos[0].partitionInfo) + } + var buffer bytes.Buffer + for index, info := range p.PartitionInfos { + if index > 0 { + buffer.WriteString(", ") } - var buffer bytes.Buffer - for index, info := range p.PartitionInfos { - if index > 0 { - buffer.WriteString(", ") - } - tblName := info.TableScan.Table.Name.O - if info.TableScan.TableAsName != nil && info.TableScan.TableAsName.O != "" { - tblName = info.TableScan.TableAsName.O - } + tblName := info.tableScan.Table.Name.O + if info.tableScan.TableAsName != nil && info.tableScan.TableAsName.O != "" { + tblName = info.tableScan.TableAsName.O + } - if info.TableScan.Table.GetPartitionInfo() == nil { - buffer.WriteString("table of ") - buffer.WriteString(tblName) - continue - } else { - buffer.WriteString(getAccessObjectForTableScan(sctx, info.TableScan, info.PartitionInfo)) - buffer.WriteString(" of ") - buffer.WriteString(tblName) - } + if info.tableScan.Table.GetPartitionInfo() == nil { + buffer.WriteString("table of ") + buffer.WriteString(tblName) + continue } - return buffer.String() + buffer.WriteString(getAccessObjectForTableScan(sctx, info.tableScan, info.partitionInfo)) + buffer.WriteString(" of ") + buffer.WriteString(tblName) } + return buffer.String() } func partitionAccessObject(sctx sessionctx.Context, tbl table.PartitionedTable, pi *model.PartitionInfo, partTable *PartitionInfo) string { diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 438b76522f3ce..23b055463ec72 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -68,9 +68,9 @@ var ( _ PhysicalPlan = &PhysicalTableSample{} ) -type TableScanAndPartitionInfo struct { - TableScan *PhysicalTableScan - PartitionInfo PartitionInfo +type tableScanAndPartitionInfo struct { + tableScan *PhysicalTableScan + partitionInfo PartitionInfo } // PhysicalTableReader is the table reader in tidb. @@ -92,7 +92,7 @@ type PhysicalTableReader struct { // Used by partition table. PartitionInfo PartitionInfo // Used by MPP, because MPP plan may contain union, it is possible that a physical table reader contains more than 1 table scan - PartitionInfos []TableScanAndPartitionInfo + PartitionInfos []tableScanAndPartitionInfo } // PartitionInfo indicates partition helper info in physical plan. diff --git a/planner/core/task.go b/planner/core/task.go index b5d595f6f28e6..187140c613aa5 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -2257,7 +2257,7 @@ func (t *mppTask) convertToRootTask(ctx sessionctx.Context) *rootTask { func collectPartitionInfosFromMPPPlan(p *PhysicalTableReader, mppPlan PhysicalPlan) { switch x := mppPlan.(type) { case *PhysicalTableScan: - p.PartitionInfos = append(p.PartitionInfos, TableScanAndPartitionInfo{x, x.PartitionInfo}) + p.PartitionInfos = append(p.PartitionInfos, tableScanAndPartitionInfo{x, x.PartitionInfo}) default: for _, ch := range mppPlan.Children() { collectPartitionInfosFromMPPPlan(p, ch) From 09f90d28f34a1e00d8880a749e79b111eb546138 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 10:32:56 +0800 Subject: [PATCH 5/8] fix tests --- planner/core/explain.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/planner/core/explain.go b/planner/core/explain.go index 380f1b4dfd644..26bfa775fc417 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -359,6 +359,16 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { if len(p.PartitionInfos) == 1 { return getAccessObjectForTableScan(sctx, p.PartitionInfos[0].tableScan, p.PartitionInfos[0].partitionInfo) } + containsPartitionTable := false + for _, info := range p.PartitionInfos { + if info.tableScan.Table.GetPartitionInfo() != nil { + containsPartitionTable = true + break + } + } + if !containsPartitionTable { + return "" + } var buffer bytes.Buffer for index, info := range p.PartitionInfos { if index > 0 { From 18eb2dfc12b79ec4e597cd064b1a3d4ac682cb48 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 10:35:13 +0800 Subject: [PATCH 6/8] update comments --- planner/core/physical_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 23b055463ec72..17e84b6efcf47 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -91,7 +91,7 @@ type PhysicalTableReader struct { // Used by partition table. PartitionInfo PartitionInfo - // Used by MPP, because MPP plan may contain union, it is possible that a physical table reader contains more than 1 table scan + // Used by MPP, because MPP plan may contain join/union/union all, it is possible that a physical table reader contains more than 1 table scan PartitionInfos []tableScanAndPartitionInfo } From 58b83568edd88e052a651e1c4fcdb9bbe038ac98 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 12:47:48 +0800 Subject: [PATCH 7/8] address comments --- executor/partition_table_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 71203e2f1430e..f66b1205e6a6c 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -1747,7 +1747,7 @@ func (s *partitionTableSuite) TestMPPQueryExplainInfo(c *C) { c.Assert(err == nil, IsTrue) } err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) - c.Assert(err == nil, IsTrue) + c.Assert(err, IsNil) tk.MustExec(`insert into t values (2), (7), (12)`) tk.MustExec("set tidb_enforce_mpp=1") tk.MustPartition(`select * from t where a < 3`, "p0").Sort().Check(testkit.Rows("2")) From a8000ecc656fd7f589e6a21c0d2ac3d6f7049413 Mon Sep 17 00:00:00 2001 From: xufei Date: Fri, 3 Dec 2021 14:27:10 +0800 Subject: [PATCH 8/8] fix --- executor/partition_table_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index f66b1205e6a6c..16ea8aab533f0 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -1744,7 +1744,7 @@ func (s *partitionTableSuite) TestMPPQueryExplainInfo(c *C) { tb := testGetTableByName(c, tk.Se, "tiflash_partition_test", "t") for _, partition := range tb.Meta().GetPartitionInfo().Definitions { err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, partition.ID, true) - c.Assert(err == nil, IsTrue) + c.Assert(err, IsNil) } err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) c.Assert(err, IsNil)