Skip to content

Commit

Permalink
planner: push aggregation operators down to projection by default (#2…
Browse files Browse the repository at this point in the history 
…2090)
  • Loading branch information
@TszKitLo40
TszKitLo40 authored Mar 17, 2021
1 parent 5b892a8 commit 9f7ed0f
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 101 deletions.
98 changes: 48 additions & 50 deletions cmd/explaintest/r/tpch.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,8 +447,8 @@ l_year;
id estRows task access object operator info
Sort 769.96 root tpch.nation.n_name, tpch.nation.n_name, Column#50
└─Projection 769.96 root tpch.nation.n_name, tpch.nation.n_name, Column#50, Column#52
└─HashAgg 769.96 root group by:Column#50, tpch.nation.n_name, tpch.nation.n_name, funcs:sum(Column#51)->Column#52, funcs:firstrow(tpch.nation.n_name)->tpch.nation.n_name, funcs:firstrow(tpch.nation.n_name)->tpch.nation.n_name, funcs:firstrow(Column#50)->Column#50
└─Projection 1957240.42 root tpch.nation.n_name, tpch.nation.n_name, extract(YEAR, tpch.lineitem.l_shipdate)->Column#50, mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#51
└─HashAgg 769.96 root group by:Column#59, Column#60, Column#61, funcs:sum(Column#55)->Column#52, funcs:firstrow(Column#56)->tpch.nation.n_name, funcs:firstrow(Column#57)->tpch.nation.n_name, funcs:firstrow(Column#58)->Column#50
└─Projection 1957240.42 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#55, tpch.nation.n_name, tpch.nation.n_name, extract(YEAR, tpch.lineitem.l_shipdate)->Column#58, tpch.nation.n_name, tpch.nation.n_name, extract(YEAR, tpch.lineitem.l_shipdate)->Column#61
└─Projection 1957240.42 root tpch.lineitem.l_extendedprice, tpch.lineitem.l_discount, tpch.lineitem.l_shipdate, tpch.nation.n_name, tpch.nation.n_name
└─HashJoin 1957240.42 root inner join, equal:[eq(tpch.customer.c_nationkey, tpch.nation.n_nationkey)], other cond:or(and(eq(tpch.nation.n_name, "JAPAN"), eq(tpch.nation.n_name, "INDIA")), and(eq(tpch.nation.n_name, "INDIA"), eq(tpch.nation.n_name, "JAPAN")))
├─TableReader(Build) 2.00 root data:Selection
Expand Down Expand Up @@ -521,36 +521,35 @@ id estRows task access object operator info
Sort 719.02 root Column#62
└─Projection 719.02 root Column#62, div(Column#64, Column#65)->Column#66
└─HashAgg 719.02 root group by:Column#78, funcs:sum(Column#75)->Column#64, funcs:sum(Column#76)->Column#65, funcs:firstrow(Column#77)->Column#62
└─Projection 563136.02 root case(eq(tpch.nation.n_name, INDIA), Column#63, 0)->Column#75, Column#63, Column#62, Column#62
└─Projection 563136.02 root extract(YEAR, tpch.orders.o_orderdate)->Column#62, mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#63, tpch.nation.n_name
└─Projection 563136.02 root tpch.lineitem.l_extendedprice, tpch.lineitem.l_discount, tpch.orders.o_orderdate, tpch.nation.n_name
└─HashJoin 563136.02 root inner join, equal:[eq(tpch.supplier.s_nationkey, tpch.nation.n_nationkey)]
├─TableReader(Build) 25.00 root data:TableFullScan
│ └─TableFullScan 25.00 cop[tikv] table:n2 keep order:false
└─HashJoin(Probe) 563136.02 root inner join, equal:[eq(tpch.lineitem.l_suppkey, tpch.supplier.s_suppkey)]
├─TableReader(Build) 500000.00 root data:TableFullScan
│ └─TableFullScan 500000.00 cop[tikv] table:supplier keep order:false
└─HashJoin(Probe) 563136.02 root inner join, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)]
├─TableReader(Build) 61674.00 root data:Selection
│ └─Selection 61674.00 cop[tikv] eq(tpch.part.p_type, "SMALL PLATED COPPER")
│ └─TableFullScan 10000000.00 cop[tikv] table:part keep order:false
└─IndexHashJoin(Probe) 90788402.51 root inner join, inner:IndexLookUp, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.orders.o_orderkey, tpch.lineitem.l_orderkey)
├─HashJoin(Build) 22413367.93 root inner join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)]
│ ├─HashJoin(Build) 1500000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)]
│ │ ├─HashJoin(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)]
│ │ │ ├─TableReader(Build) 1.00 root data:Selection
│ │ │ │ └─Selection 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA")
│ │ │ │ └─TableFullScan 5.00 cop[tikv] table:region keep order:false
│ │ │ └─TableReader(Probe) 25.00 root data:TableFullScan
│ │ │ └─TableFullScan 25.00 cop[tikv] table:n1 keep order:false
│ │ └─TableReader(Probe) 7500000.00 root data:TableFullScan
│ │ └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
│ └─TableReader(Probe) 22413367.93 root data:Selection
│ └─Selection 22413367.93 cop[tikv] ge(tpch.orders.o_orderdate, 1995-01-01 00:00:00.000000), le(tpch.orders.o_orderdate, 1996-12-31 00:00:00.000000)
│ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
└─IndexLookUp(Probe) 4.05 root
├─IndexRangeScan(Build) 4.05 cop[tikv] table:lineitem, index:PRIMARY(L_ORDERKEY, L_LINENUMBER) range: decided by [eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)], keep order:false
└─TableRowIDScan(Probe) 4.05 cop[tikv] table:lineitem keep order:false
└─Projection 563136.02 root case(eq(tpch.nation.n_name, INDIA), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), 0)->Column#75, mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#76, extract(YEAR, tpch.orders.o_orderdate)->Column#77, extract(YEAR, tpch.orders.o_orderdate)->Column#78
└─Projection 563136.02 root tpch.lineitem.l_extendedprice, tpch.lineitem.l_discount, tpch.orders.o_orderdate, tpch.nation.n_name
└─HashJoin 563136.02 root inner join, equal:[eq(tpch.supplier.s_nationkey, tpch.nation.n_nationkey)]
├─TableReader(Build) 25.00 root data:TableFullScan
│ └─TableFullScan 25.00 cop[tikv] table:n2 keep order:false
└─HashJoin(Probe) 563136.02 root inner join, equal:[eq(tpch.lineitem.l_suppkey, tpch.supplier.s_suppkey)]
├─TableReader(Build) 500000.00 root data:TableFullScan
│ └─TableFullScan 500000.00 cop[tikv] table:supplier keep order:false
└─HashJoin(Probe) 563136.02 root inner join, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)]
├─TableReader(Build) 61674.00 root data:Selection
│ └─Selection 61674.00 cop[tikv] eq(tpch.part.p_type, "SMALL PLATED COPPER")
│ └─TableFullScan 10000000.00 cop[tikv] table:part keep order:false
└─IndexHashJoin(Probe) 90788402.51 root inner join, inner:IndexLookUp, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.orders.o_orderkey, tpch.lineitem.l_orderkey)
├─HashJoin(Build) 22413367.93 root inner join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)]
│ ├─HashJoin(Build) 1500000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)]
│ │ ├─HashJoin(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)]
│ │ │ ├─TableReader(Build) 1.00 root data:Selection
│ │ │ │ └─Selection 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA")
│ │ │ │ └─TableFullScan 5.00 cop[tikv] table:region keep order:false
│ │ │ └─TableReader(Probe) 25.00 root data:TableFullScan
│ │ │ └─TableFullScan 25.00 cop[tikv] table:n1 keep order:false
│ │ └─TableReader(Probe) 7500000.00 root data:TableFullScan
│ │ └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
│ └─TableReader(Probe) 22413367.93 root data:Selection
│ └─Selection 22413367.93 cop[tikv] ge(tpch.orders.o_orderdate, 1995-01-01 00:00:00.000000), le(tpch.orders.o_orderdate, 1996-12-31 00:00:00.000000)
│ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
└─IndexLookUp(Probe) 4.05 root
├─IndexRangeScan(Build) 4.05 cop[tikv] table:lineitem, index:PRIMARY(L_ORDERKEY, L_LINENUMBER) range: decided by [eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)], keep order:false
└─TableRowIDScan(Probe) 4.05 cop[tikv] table:lineitem keep order:false
/*
Q9 Product Type Profit Measure Query
This query determines how much profit is made on a given line of parts, broken out by supplier nation and year.
Expand Down Expand Up @@ -597,8 +596,8 @@ o_year desc;
id estRows task access object operator info
Sort 2406.00 root tpch.nation.n_name, Column#53:desc
└─Projection 2406.00 root tpch.nation.n_name, Column#53, Column#55
└─HashAgg 2406.00 root group by:Column#53, tpch.nation.n_name, funcs:sum(Column#54)->Column#55, funcs:firstrow(tpch.nation.n_name)->tpch.nation.n_name, funcs:firstrow(Column#53)->Column#53
└─Projection 241379546.70 root tpch.nation.n_name, extract(YEAR, tpch.orders.o_orderdate)->Column#53, minus(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), mul(tpch.partsupp.ps_supplycost, tpch.lineitem.l_quantity))->Column#54
└─HashAgg 2406.00 root group by:Column#67, Column#68, funcs:sum(Column#64)->Column#55, funcs:firstrow(Column#65)->tpch.nation.n_name, funcs:firstrow(Column#66)->Column#53
└─Projection 241379546.70 root minus(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), mul(tpch.partsupp.ps_supplycost, tpch.lineitem.l_quantity))->Column#64, tpch.nation.n_name, extract(YEAR, tpch.orders.o_orderdate)->Column#66, tpch.nation.n_name, extract(YEAR, tpch.orders.o_orderdate)->Column#68
└─Projection 241379546.70 root tpch.lineitem.l_quantity, tpch.lineitem.l_extendedprice, tpch.lineitem.l_discount, tpch.partsupp.ps_supplycost, tpch.orders.o_orderdate, tpch.nation.n_name
└─HashJoin 241379546.70 root inner join, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)]
├─TableReader(Build) 75000000.00 root data:TableFullScan
Expand Down Expand Up @@ -1160,20 +1159,19 @@ Sort 20000.00 root tpch.supplier.s_name
│ └─TableReader(Probe) 500000.00 root data:TableFullScan
│ └─TableFullScan 500000.00 cop[tikv] table:supplier keep order:false
└─HashAgg(Probe) 257492.04 root group by:tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_suppkey)->tpch.partsupp.ps_suppkey
└─Projection 257492.04 root tpch.partsupp.ps_suppkey
└─Selection 257492.04 root gt(cast(tpch.partsupp.ps_availqty, decimal(20,0) BINARY), mul(0.5, Column#44))
└─HashAgg 321865.05 root group by:tpch.partsupp.ps_partkey, tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_suppkey)->tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_availqty)->tpch.partsupp.ps_availqty, funcs:sum(tpch.lineitem.l_quantity)->Column#44
└─HashJoin 9711455.06 root left outer join, equal:[eq(tpch.partsupp.ps_partkey, tpch.lineitem.l_partkey) eq(tpch.partsupp.ps_suppkey, tpch.lineitem.l_suppkey)]
├─IndexHashJoin(Build) 321865.05 root inner join, inner:IndexLookUp, outer key:tpch.part.p_partkey, inner key:tpch.partsupp.ps_partkey, equal cond:eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey)
│ ├─TableReader(Build) 80007.93 root data:Selection
│ │ └─Selection 80007.93 cop[tikv] like(tpch.part.p_name, "green%", 92)
│ │ └─TableFullScan 10000000.00 cop[tikv] table:part keep order:false
│ └─IndexLookUp(Probe) 4.02 root
│ ├─IndexRangeScan(Build) 4.02 cop[tikv] table:partsupp, index:PRIMARY(PS_PARTKEY, PS_SUPPKEY) range: decided by [eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)], keep order:false
│ └─TableRowIDScan(Probe) 4.02 cop[tikv] table:partsupp keep order:false
└─TableReader(Probe) 44189356.65 root data:Selection
└─Selection 44189356.65 cop[tikv] ge(tpch.lineitem.l_shipdate, 1993-01-01 00:00:00.000000), lt(tpch.lineitem.l_shipdate, 1994-01-01)
└─TableFullScan 300005811.00 cop[tikv] table:lineitem keep order:false
└─Selection 257492.04 root gt(cast(tpch.partsupp.ps_availqty, decimal(20,0) BINARY), mul(0.5, Column#44))
└─HashAgg 321865.05 root group by:tpch.partsupp.ps_partkey, tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_suppkey)->tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_availqty)->tpch.partsupp.ps_availqty, funcs:sum(tpch.lineitem.l_quantity)->Column#44
└─HashJoin 9711455.06 root left outer join, equal:[eq(tpch.partsupp.ps_partkey, tpch.lineitem.l_partkey) eq(tpch.partsupp.ps_suppkey, tpch.lineitem.l_suppkey)]
├─IndexHashJoin(Build) 321865.05 root inner join, inner:IndexLookUp, outer key:tpch.part.p_partkey, inner key:tpch.partsupp.ps_partkey, equal cond:eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey)
│ ├─TableReader(Build) 80007.93 root data:Selection
│ │ └─Selection 80007.93 cop[tikv] like(tpch.part.p_name, "green%", 92)
│ │ └─TableFullScan 10000000.00 cop[tikv] table:part keep order:false
│ └─IndexLookUp(Probe) 4.02 root
│ ├─IndexRangeScan(Build) 4.02 cop[tikv] table:partsupp, index:PRIMARY(PS_PARTKEY, PS_SUPPKEY) range: decided by [eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)], keep order:false
│ └─TableRowIDScan(Probe) 4.02 cop[tikv] table:partsupp keep order:false
└─TableReader(Probe) 44189356.65 root data:Selection
└─Selection 44189356.65 cop[tikv] ge(tpch.lineitem.l_shipdate, 1993-01-01 00:00:00.000000), lt(tpch.lineitem.l_shipdate, 1994-01-01)
└─TableFullScan 300005811.00 cop[tikv] table:lineitem keep order:false
/*
Q21 Suppliers Who Kept Orders Waiting Query
This query identifies certain suppliers who were not able to ship required parts in a timely manner.
Expand Down Expand Up @@ -1298,8 +1296,8 @@ cntrycode;
id estRows task access object operator info
Sort 1.00 root Column#27
└─Projection 1.00 root Column#27, Column#28, Column#29
└─HashAgg 1.00 root group by:Column#27, funcs:count(1)->Column#28, funcs:sum(tpch.customer.c_acctbal)->Column#29, funcs:firstrow(Column#27)->Column#27
└─Projection 0.00 root substring(tpch.customer.c_phone, 1, 2)->Column#27, tpch.customer.c_acctbal
└─HashAgg 1.00 root group by:Column#33, funcs:count(1)->Column#28, funcs:sum(Column#31)->Column#29, funcs:firstrow(Column#32)->Column#27
└─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#32, substring(tpch.customer.c_phone, 1, 2)->Column#33
└─HashJoin 0.00 root anti semi join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)]
├─TableReader(Build) 75000000.00 root data:TableFullScan
│ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
Expand Down
2 changes: 1 addition & 1 deletion planner/core/rule_aggregation_push_down.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e
p = proj
}
}
} else if proj, ok1 := child.(*LogicalProjection); ok1 && p.SCtx().GetSessionVars().AllowAggPushDown {
} else if proj, ok1 := child.(*LogicalProjection); ok1 {
// TODO: This optimization is not always reasonable. We have not supported pushing projection to kv layer yet,
// so we must do this optimization.
for i, gbyItem := range agg.GroupByItems {
Expand Down
Loading

0 comments on commit 9f7ed0f

Please sign in to comment.