Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner: eliminate empty projection below aggregation (#29606) #29635

Merged
merged 5 commits into from
Dec 19, 2021
Merged

planner: eliminate empty projection below aggregation (#29606) #29635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b00bfcd
cherry pick #29606 to release-5.0
time-and-fate Nov 10, 2021
e771986
Merge branch 'release-5.0' into release-5.0-82f2bccf0303
time-and-fate Dec 17, 2021
37c98ce
Merge branch 'release-5.0' into release-5.0-82f2bccf0303
LittleFall Dec 17, 2021
c443870
Merge branch 'release-5.0' into release-5.0-82f2bccf0303
ti-chi-bot Dec 19, 2021
aa3e303
fix
time-and-fate Dec 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions planner/core/logical_plan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,9 @@ func (s *testPlanSuite) TestProjectionEliminator(c *C) {
{
sql: "select 1+num from (select 1+a as num from t) t1;",
best: "DataScan(t)->Projection",
}, {
sql: "select count(*) from t where a in (select b from t2 where a is null);",
best: "Join{DataScan(t)->Dual->Aggr(firstrow(test.t2.b))}(test.t.a,test.t2.b)->Aggr(count(1))->Projection",
},
}

Expand All @@ -619,6 +622,29 @@ func (s *testPlanSuite) TestProjectionEliminator(c *C) {
}
}

func (s *testPlanSuite) TestCS3389(c *C) {
defer testleak.AfterTest(c)()

ctx := context.Background()
stmt, err := s.ParseOneStmt("select count(*) from t where a in (select b from t2 where a is null);", "", "")
c.Assert(err, IsNil)
p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is)
c.Assert(err, IsNil)
p, err = logicalOptimize(context.TODO(), flagBuildKeyInfo|flagPrunColumns|flagPrunColumnsAgain|flagEliminateProjection|flagJoinReOrder, p.(LogicalPlan))
c.Assert(err, IsNil)

// Assert that all Projection is not empty and there is no Projection between Aggregation and Join.
proj, isProj := p.(*LogicalProjection)
c.Assert(isProj, IsTrue)
c.Assert(len(proj.Exprs) > 0, IsTrue)
child := proj.Children()[0]
agg, isAgg := child.(*LogicalAggregation)
c.Assert(isAgg, IsTrue)
child = agg.Children()[0]
_, isJoin := child.(*LogicalJoin)
c.Assert(isJoin, IsTrue)
}

func (s *testPlanSuite) TestAllocID(c *C) {
ctx := MockContext()
pA := DataSource{}.Init(ctx, 0)
Expand Down
16 changes: 15 additions & 1 deletion planner/core/rule_column_pruning.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,21 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column)
la.GroupByItems = []expression.Expression{expression.NewOne()}
}
}
return child.PruneColumns(selfUsedCols)
err := child.PruneColumns(selfUsedCols)
if err != nil {
return err
}
// Do an extra Projection Elimination here. This is specially for empty Projection below Aggregation.
// This kind of Projection would cause some bugs for MPP plan and is safe to be removed.
// This kind of Projection should be removed in Projection Elimination, but currently PrunColumnsAgain is
// the last rule. So we specially handle this case here.
if childProjection, isProjection := child.(*LogicalProjection); isProjection {
if len(childProjection.Exprs) == 0 && childProjection.Schema().Len() == 0 {
childOfChild := childProjection.children[0]
la.SetChildren(childOfChild)
}
}
return nil
}

func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*expression.Column) {
Expand Down
104 changes: 50 additions & 54 deletions planner/core/testdata/integration_serial_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,59 +67,56 @@
{
"SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k",
"Plan": [
"HashAgg 1.00 root funcs:count(Column#12)->Column#11",
"└─TableReader 1.00 root data:ExchangeSender",
" └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough",
" └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12",
" └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false"
"StreamAgg 1.00 root funcs:count(1)->Column#11",
"└─TableReader 8.00 root data:ExchangeSender",
" └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough",
" └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false"
]
},
{
"SQL": "explain format = 'brief' select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k",
"Plan": [
"HashAgg 1.00 root funcs:count(Column#18)->Column#17",
"└─TableReader 1.00 root data:ExchangeSender",
" └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough",
" └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#18",
" └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d3_t.d3_k))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d3_t keep order:false",
" └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d2_t.d2_k))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d2_t keep order:false",
" └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))",
" └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false"
"StreamAgg 1.00 root funcs:count(1)->Column#17",
"└─TableReader 8.00 root data:ExchangeSender",
" └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough",
" └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d3_t keep order:false",
" └─HashJoin(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d2_t keep order:false",
" └─HashJoin(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))",
" └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false"
]
},
{
"SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k",
"Plan": [
"HashAgg 1.00 root funcs:count(Column#12)->Column#11",
"└─TableReader 1.00 root data:ExchangeSender",
" └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough",
" └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12",
" └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false"
"StreamAgg 1.00 root funcs:count(1)->Column#11",
"└─TableReader 8.00 root data:ExchangeSender",
" └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough",
" └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false"
]
},
{
Expand Down Expand Up @@ -153,17 +150,16 @@
{
"SQL": "explain format = 'brief' select count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value",
"Plan": [
"HashAgg 1.00 root funcs:count(Column#12)->Column#11",
"└─TableReader 1.00 root data:ExchangeSender",
" └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough",
" └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12",
" └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)",
" ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ",
" │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))",
" │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false"
"StreamAgg 1.00 root funcs:count(1)->Column#11",
"└─TableReader 8.00 root data:ExchangeSender",
" └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough",
" └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)",
" ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ",
" │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast",
" │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))",
" │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false",
" └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))",
" └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false"
]
},
{
Expand Down