Skip to content

Commit 17a1ce5

Browse files
authored
[fix](nereids) add a project node above sort node to eliminate unused order by keys (apache#17913)
if the order by keys are not simple slot in sort node, the order by exprs have to been added to sort node's output tuple. In that case, we need add a project node above sort node to eliminate the unused order by exprs. for example: ```sql WITH t0 AS (SELECT DATE_FORMAT(date, '%Y%m%d') AS date FROM cir_1756_t1 ), t3 AS (SELECT date_format(date, '%Y%m%d') AS `date` FROM `cir_1756_t2` GROUP BY date_format(date, '%Y%m%d') **ORDER BY date_format(date, '%Y%m%d')** ) SELECT t0.date FROM t0 LEFT JOIN t3 ON t0.date = t3.date; ``` before: ``` +--------------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String | +--------------------------------------------------------------------------------------------------------------------------------------------------+ | LogicalProject[159] ( distinct=false, projects=[date#1], excepts=[], canEliminate=true ) | | +--LogicalJoin[158] ( type=LEFT_OUTER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=[(date#1 = date#3)], otherJoinConjuncts=[] ) | | |--LogicalProject[151] ( distinct=false, projects=[date_format(date#0, '%Y%m%d') AS `date`#1], excepts=[], canEliminate=true ) | | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t1, indexName=cir_1756_t1, selectedIndexId=412339, preAgg=ON ) | | +--LogicalSort[157] ( orderKeys=[date_format(cast(date#3 as DATETIME), '%Y%m%d') asc null first] ) | | +--LogicalAggregate[156] ( groupByExpr=[date#3], outputExpr=[date#3], hasRepeat=false ) | | +--LogicalProject[155] ( distinct=false, projects=[date_format(date#2, '%Y%m%d') AS `date`#3], excepts=[], canEliminate=true ) | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t2, indexName=cir_1756_t2, selectedIndexId=412352, preAgg=ON ) | +--------------------------------------------------------------------------------------------------------------------------------------------------+ ``` after: ``` +--------------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String | +--------------------------------------------------------------------------------------------------------------------------------------------------+ | LogicalProject[171] ( distinct=false, projects=[date#2], excepts=[], canEliminate=true ) | | +--LogicalJoin[170] ( type=LEFT_OUTER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=[(date#2 = date#4)], otherJoinConjuncts=[] ) | | |--LogicalProject[162] ( distinct=false, projects=[date_format(date#0, '%Y%m%d') AS `date`#2], excepts=[], canEliminate=true ) | | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t1, indexName=cir_1756_t1, selectedIndexId=1049812, preAgg=ON ) | | +--LogicalProject[169] ( distinct=false, projects=[date#4], excepts=[], canEliminate=false ) | | +--LogicalSort[168] ( orderKeys=[date_format(cast(date#4 as DATETIME), '%Y%m%d') asc null first] ) | | +--LogicalAggregate[167] ( groupByExpr=[date#4], outputExpr=[date#4], hasRepeat=false ) | | +--LogicalProject[166] ( distinct=false, projects=[date_format(date#3, '%Y%m%d') AS `date`#4], excepts=[], canEliminate=true ) | | +--LogicalOlapScan ( qualified=default_cluster:bugfix.cir_1756_t2, indexName=cir_1756_t2, selectedIndexId=1049825, preAgg=ON ) | +--------------------------------------------------------------------------------------------------------------------------------------------------+ ```
1 parent 6cbf393 commit 17a1ce5

16 files changed

+182
-22
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java

+1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ public enum RuleType {
133133
COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE),
134134
PRUNE_ONE_ROW_RELATION_COLUMN(RuleTypeClass.REWRITE),
135135
COLUMN_PRUNE_SORT_CHILD(RuleTypeClass.REWRITE),
136+
COLUMN_PRUNE_SORT(RuleTypeClass.REWRITE),
136137
COLUMN_PRUNE_JOIN_CHILD(RuleTypeClass.REWRITE),
137138
COLUMN_PRUNE_REPEAT_CHILD(RuleTypeClass.REWRITE),
138139
// expression of plan rewrite

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public Rule build() {
130130
if (projects.equals(newProjects)) {
131131
return project;
132132
}
133-
return new LogicalProject<>(newProjects, project.child());
133+
return project.withProjectsAndChild(newProjects, project.child());
134134
}).toRule(RuleType.REWRITE_PROJECT_EXPRESSION);
135135
}
136136
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public List<Rule> buildRules() {
3535
new PruneFilterChildColumns().build(),
3636
new PruneAggChildColumns().build(),
3737
new PruneJoinChildrenColumns().build(),
38+
new PruneSortColumns().build(),
3839
new PruneSortChildColumns().build(),
3940
new MergeProjects().build(),
4041
new PruneRepeatChildColumns().build()

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpression.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
2929
import org.apache.doris.nereids.trees.plans.Plan;
3030
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
31-
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
3231
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
3332
import org.apache.doris.nereids.util.ExpressionUtils;
3433

@@ -65,10 +64,11 @@ public Rule build() {
6564
boolean needAggregate = bottomProjects.stream().anyMatch(expr ->
6665
expr.anyMatch(AggregateFunction.class::isInstance));
6766
if (needAggregate) {
68-
normalizedChild = new LogicalAggregate<>(
69-
ImmutableList.of(), ImmutableList.copyOf(bottomProjects), project.child());
67+
normalizedChild = new LogicalAggregate<>(ImmutableList.of(),
68+
ImmutableList.copyOf(bottomProjects), project.child());
7069
} else {
71-
normalizedChild = new LogicalProject<>(ImmutableList.copyOf(bottomProjects), project.child());
70+
normalizedChild = project.withProjectsAndChild(
71+
ImmutableList.copyOf(bottomProjects), project.child());
7272
}
7373
}
7474

@@ -89,7 +89,7 @@ public Rule build() {
8989

9090
// 3. handle top projects
9191
List<NamedExpression> topProjects = ctxForWindows.normalizeToUseSlotRef(normalizedOutputs1);
92-
return new LogicalProject<>(topProjects, normalizedLogicalWindow);
92+
return project.withProjectsAndChild(topProjects, normalizedLogicalWindow);
9393
}).toRule(RuleType.EXTRACT_AND_NORMALIZE_WINDOW_EXPRESSIONS);
9494
}
9595

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.doris.nereids.rules.RuleType;
2222
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
2323
import org.apache.doris.nereids.trees.expressions.NamedExpression;
24-
import org.apache.doris.nereids.trees.plans.Plan;
2524
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
2625

2726
import java.util.List;
@@ -47,9 +46,10 @@ public class MergeProjects extends OneRewriteRuleFactory {
4746
@Override
4847
public Rule build() {
4948
return logicalProject(logicalProject()).then(project -> {
50-
LogicalProject<Plan> childProject = project.child();
49+
LogicalProject childProject = project.child();
5150
List<NamedExpression> projectExpressions = project.mergeProjections(childProject);
52-
return new LogicalProject<>(projectExpressions, childProject.child(0));
51+
LogicalProject newProject = childProject.canEliminate() ? project : childProject;
52+
return newProject.withProjectsAndChild(projectExpressions, childProject.child(0));
5353
}).toRule(RuleType.MERGE_PROJECTS);
5454
}
5555
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.rewrite.logical;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
24+
25+
import com.google.common.collect.ImmutableList;
26+
27+
import java.util.stream.Collectors;
28+
29+
/**
30+
the sort node will create new slots for order by keys if the order by keys is not in the output
31+
so need create a project above sort node to prune the unnecessary order by keys
32+
*/
33+
public class PruneSortColumns extends OneRewriteRuleFactory {
34+
@Override
35+
public Rule build() {
36+
return logicalSort()
37+
.when(sort -> !sort.isOrderKeysPruned() && !sort.getOutputSet()
38+
.containsAll(sort.getOrderKeys().stream()
39+
.map(orderKey -> orderKey.getExpr()).collect(Collectors.toSet())))
40+
.then(sort -> {
41+
return new LogicalProject(sort.getOutput(), ImmutableList.of(), false,
42+
sort.withOrderKeysPruned(true));
43+
}).toRule(RuleType.COLUMN_PRUNE_SORT);
44+
}
45+
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public Rule build() {
7575
}
7676
});
7777

78-
LogicalProject newProject = new LogicalProject<>(newProjects, filter.child());
78+
LogicalProject newProject = project.withProjectsAndChild(newProjects, filter.child());
7979
LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject);
8080
LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newFilter));
8181
return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(),

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public Rule build() {
6464
if (apply.getSubqueryExpr() instanceof ScalarSubquery) {
6565
newProjects.add(project.getProjects().get(0));
6666
}
67-
return new LogicalProject(newProjects, newCorrelate);
67+
return project.withProjectsAndChild(newProjects, newCorrelate);
6868
}).toRule(RuleType.PULL_UP_PROJECT_UNDER_APPLY);
6969
}
7070
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,12 @@ public Rule build() {
9090
if (leftOutput.equals(leftProjects)) {
9191
left = join.left();
9292
} else {
93-
left = new LogicalProject<>(leftProjects, join.left());
93+
left = project.withProjectsAndChild(leftProjects, join.left());
9494
}
9595
if (rightOutput.equals(rightProjects)) {
9696
right = join.right();
9797
} else {
98-
right = new LogicalProject<>(rightProjects, join.right());
98+
right = project.withProjectsAndChild(rightProjects, join.right());
9999
}
100100

101101
// If condition use alias slot, we should replace condition
@@ -109,7 +109,7 @@ public Rule build() {
109109
List<Expression> newOther = replaceJoinConjuncts(join.getOtherJoinConjuncts(), replaceMap);
110110

111111
Plan newJoin = join.withConjunctsChildren(newHash, newOther, left, right);
112-
return new LogicalProject<>(newProjects, newJoin);
112+
return project.withProjectsAndChild(newProjects, newJoin);
113113
}).toRule(RuleType.PUSHDOWN_ALIAS_THROUGH_JOIN);
114114
}
115115

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public List<Rule> buildRules() {
4444
RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT.build(
4545
logicalFilter(logicalProject()).then(filter -> {
4646
LogicalProject<Plan> project = filter.child();
47-
return new LogicalProject<>(
47+
return project.withProjectsAndChild(
4848
project.getProjects(),
4949
new LogicalFilter<>(
5050
ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()),
@@ -60,7 +60,7 @@ public List<Rule> buildRules() {
6060
LogicalLimit<LogicalProject<Plan>> limit = filter.child();
6161
LogicalProject<Plan> project = limit.child();
6262

63-
return new LogicalProject<>(
63+
return project.withProjectsAndChild(
6464
project.getProjects(),
6565
new LogicalFilter<>(
6666
ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()),

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public Rule build() {
5252
LogicalProject<LogicalLimit<Plan>> logicalProject = ctx.root;
5353
LogicalLimit<Plan> logicalLimit = logicalProject.child();
5454
return new LogicalLimit<>(logicalLimit.getLimit(), logicalLimit.getOffset(),
55-
logicalLimit.getPhase(), new LogicalProject<>(logicalProject.getProjects(),
55+
logicalLimit.getPhase(), logicalProject.withProjectsAndChild(logicalProject.getProjects(),
5656
logicalLimit.child()));
5757
}).toRule(RuleType.PUSHDOWN_PROJECT_THROUGH_LIMIT);
5858
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public Rule build() {
8787
.filter(e -> !projects.contains(e))
8888
.map(NamedExpression.class::cast)
8989
.forEach(projects::add);
90-
LogicalProject newProject = new LogicalProject(projects, child);
90+
LogicalProject newProject = project.withProjectsAndChild(projects, child);
9191
return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(),
9292
ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(),
9393
apply.getSubCorrespondingConject(),

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java

+5
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,11 @@ public LogicalProject<Plan> withChildren(List<Plan> children) {
175175
return new LogicalProject<>(projects, excepts, canEliminate, children.get(0), isDistinct);
176176
}
177177

178+
public LogicalProject<Plan> withProjectsAndChild(List<NamedExpression> projects, Plan child) {
179+
return new LogicalProject<>(projects, excepts, canEliminate,
180+
Optional.empty(), Optional.empty(), child, isDistinct);
181+
}
182+
178183
@Override
179184
public LogicalProject<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
180185
return new LogicalProject<>(projects, excepts, canEliminate,

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java

+26-4
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,29 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
4646

4747
private final List<OrderKey> orderKeys;
4848

49+
private final boolean orderKeysPruned;
50+
4951
public LogicalSort(List<OrderKey> orderKeys, CHILD_TYPE child) {
5052
this(orderKeys, Optional.empty(), Optional.empty(), child);
5153
}
5254

55+
public LogicalSort(List<OrderKey> orderKeys, CHILD_TYPE child, boolean orderKeysPruned) {
56+
this(orderKeys, Optional.empty(), Optional.empty(), child, orderKeysPruned);
57+
}
58+
5359
/**
5460
* Constructor for LogicalSort.
5561
*/
5662
public LogicalSort(List<OrderKey> orderKeys, Optional<GroupExpression> groupExpression,
5763
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
64+
this(orderKeys, groupExpression, logicalProperties, child, false);
65+
}
66+
67+
public LogicalSort(List<OrderKey> orderKeys, Optional<GroupExpression> groupExpression,
68+
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child, boolean orderKeysPruned) {
5869
super(PlanType.LOGICAL_SORT, groupExpression, logicalProperties, child);
5970
this.orderKeys = ImmutableList.copyOf(Objects.requireNonNull(orderKeys, "orderKeys can not be null"));
71+
this.orderKeysPruned = orderKeysPruned;
6072
}
6173

6274
@Override
@@ -68,6 +80,10 @@ public List<OrderKey> getOrderKeys() {
6880
return orderKeys;
6981
}
7082

83+
public boolean isOrderKeysPruned() {
84+
return orderKeysPruned;
85+
}
86+
7187
@Override
7288
public String toString() {
7389
return Utils.toSqlString("LogicalSort[" + id.asInt() + "]",
@@ -106,21 +122,27 @@ public List<? extends Expression> getExpressions() {
106122
@Override
107123
public LogicalSort<Plan> withChildren(List<Plan> children) {
108124
Preconditions.checkArgument(children.size() == 1);
109-
return new LogicalSort<>(orderKeys, children.get(0));
125+
return new LogicalSort<>(orderKeys, children.get(0), orderKeysPruned);
110126
}
111127

112128
@Override
113129
public LogicalSort<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
114-
return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child());
130+
return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(),
131+
orderKeysPruned);
115132
}
116133

117134
@Override
118135
public LogicalSort<Plan> withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
119-
return new LogicalSort<>(orderKeys, Optional.empty(), logicalProperties, child());
136+
return new LogicalSort<>(orderKeys, Optional.empty(), logicalProperties, child(), false);
120137
}
121138

122139
public LogicalSort<Plan> withOrderKeys(List<OrderKey> orderKeys) {
123140
return new LogicalSort<>(orderKeys, Optional.empty(),
124-
Optional.of(getLogicalProperties()), child());
141+
Optional.of(getLogicalProperties()), child(), false);
142+
}
143+
144+
public LogicalSort<Plan> withOrderKeysPruned(boolean orderKeysPruned) {
145+
return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(),
146+
orderKeysPruned);
125147
}
126148
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select --
3+
20200202
4+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("inlineview_with_project") {
19+
sql "SET enable_nereids_planner=true"
20+
sql "SET enable_fallback_to_original_planner=false"
21+
sql """
22+
drop table if exists cir_1756_t1;
23+
"""
24+
25+
sql """
26+
drop table if exists cir_1756_t2;
27+
"""
28+
29+
sql """
30+
create table cir_1756_t1 (`date` date not null)
31+
ENGINE=OLAP
32+
DISTRIBUTED BY HASH(`date`) BUCKETS 5
33+
PROPERTIES (
34+
"replication_allocation" = "tag.location.default: 1",
35+
"in_memory" = "false",
36+
"storage_format" = "V2"
37+
);
38+
"""
39+
40+
sql """
41+
create table cir_1756_t2 ( `date` date not null )
42+
ENGINE=OLAP
43+
DISTRIBUTED BY HASH(`date`) BUCKETS 5
44+
PROPERTIES (
45+
"replication_allocation" = "tag.location.default: 1",
46+
"in_memory" = "false",
47+
"storage_format" = "V2"
48+
);
49+
"""
50+
51+
sql """
52+
insert into cir_1756_t1 values("2020-02-02");
53+
"""
54+
55+
sql """
56+
insert into cir_1756_t2 values("2020-02-02");
57+
"""
58+
59+
qt_select """
60+
WITH t0 AS
61+
(SELECT DATE_FORMAT(date,
62+
'%Y%m%d') AS date
63+
FROM cir_1756_t1 ), t3 AS
64+
(SELECT date_format(date,
65+
'%Y%m%d') AS `date`
66+
FROM `cir_1756_t2`
67+
GROUP BY date_format(date, '%Y%m%d')
68+
ORDER BY date_format(date, '%Y%m%d') )
69+
SELECT t0.date
70+
FROM t0
71+
LEFT JOIN t3
72+
ON t0.date = t3.date;
73+
"""
74+
75+
sql """
76+
drop table if exists cir_1756_t1;
77+
"""
78+
79+
sql """
80+
drop table if exists cir_1756_t2;
81+
"""
82+
}

0 commit comments

Comments
 (0)