Skip to content

Commit 320eda7

Browse files
authored
[fix](nereids) remove useless cast in in-predicate (apache#23171)
consider sql "select * from test_simplify_in_predicate_t where a in ('1992-01-31', '1992-02-01', '1992-02-02', '1992-02-03', '1992-02-04');" before: ``` | 0:VOlapScanNode | | TABLE: default_cluster:bugfix.test_simplify_in_predicate_t(test_simplify_in_predicate_t), PREAGGREGATION: OFF. Reason: No aggregate on scan. | | PREDICATES: CAST(a[#0] AS DATETIMEV2(0)) IN ('1992-01-31 00:00:00', '1992-02-01 00:00:00', '1992-02-02 00:00:00', '1992-02-03 00:00:00', '1992-02-04 00:00:00') AND __DORIS_DELETE_SIGN__[#1] = 0 | | partitions=0/1, tablets=0/0, tabletList= | | cardinality=1, avgRowSize=0.0, numNodes=1 | | pushAggOp=NONE | | projections: a[#0] | | project output tuple id: 1 | | tuple ids: 0 ``` after: ``` | 0:VOlapScanNode | | TABLE: default_cluster:bugfix.test_simplify_in_predicate_t(test_simplify_in_predicate_t), PREAGGREGATION: OFF. Reason: No aggregate on scan. | | PREDICATES: a[#0] IN ('1992-01-31', '1992-02-01', '1992-02-02', '1992-02-03', '1992-02-04') AND __DORIS_DELETE_SIGN__[#1] = 0 | | partitions=0/1, tablets=0/0, tabletList= | | cardinality=1, avgRowSize=0.0, numNodes=1 | | pushAggOp=NONE | | projections: a[#0] | | project output tuple id: 1 | | tuple ids: 0 ```
1 parent 6c5072f commit 320eda7

File tree

6 files changed

+179
-5
lines changed

6 files changed

+179
-5
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.nereids.rules.expression.rules.OrToIn;
2323
import org.apache.doris.nereids.rules.expression.rules.SimplifyComparisonPredicate;
2424
import org.apache.doris.nereids.rules.expression.rules.SimplifyDecimalV3Comparison;
25+
import org.apache.doris.nereids.rules.expression.rules.SimplifyInPredicate;
2526
import org.apache.doris.nereids.rules.expression.rules.SimplifyRange;
2627

2728
import com.google.common.collect.ImmutableList;
@@ -36,6 +37,7 @@ public class ExpressionOptimization extends ExpressionRewrite {
3637
ExtractCommonFactorRule.INSTANCE,
3738
DistinctPredicatesRule.INSTANCE,
3839
SimplifyComparisonPredicate.INSTANCE,
40+
SimplifyInPredicate.INSTANCE,
3941
SimplifyDecimalV3Comparison.INSTANCE,
4042
SimplifyRange.INSTANCE,
4143
OrToIn.INSTANCE
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.expression.rules;
19+
20+
import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule;
21+
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
22+
import org.apache.doris.nereids.trees.expressions.Cast;
23+
import org.apache.doris.nereids.trees.expressions.Expression;
24+
import org.apache.doris.nereids.trees.expressions.InPredicate;
25+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
26+
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
27+
28+
import com.google.common.collect.Lists;
29+
30+
import java.util.List;
31+
32+
/**
33+
* SimplifyInPredicate
34+
*/
35+
public class SimplifyInPredicate extends AbstractExpressionRewriteRule {
36+
37+
public static final SimplifyInPredicate INSTANCE = new SimplifyInPredicate();
38+
39+
@Override
40+
public Expression visitInPredicate(InPredicate expr, ExpressionRewriteContext context) {
41+
if (expr.children().size() > 1) {
42+
if (expr.getCompareExpr() instanceof Cast) {
43+
Cast cast = (Cast) expr.getCompareExpr();
44+
if (cast.child().getDataType().isDateV2Type()
45+
&& expr.child(1) instanceof DateTimeV2Literal) {
46+
List<Expression> literals = expr.children().subList(1, expr.children().size());
47+
if (literals.stream().allMatch(literal -> literal instanceof DateTimeV2Literal
48+
&& canLosslessConvertToDateV2Literal((DateTimeV2Literal) literal))) {
49+
List<Expression> children = Lists.newArrayList();
50+
children.add(cast.child());
51+
literals.stream().forEach(
52+
l -> children.add(convertToDateV2Literal((DateTimeV2Literal) l)));
53+
return expr.withChildren(children);
54+
}
55+
}
56+
}
57+
}
58+
return expr;
59+
}
60+
61+
/*
62+
derive tree:
63+
DateLiteral
64+
|
65+
+--->DateTimeLiteral
66+
| |
67+
| +----->DateTimeV2Literal
68+
+--->DateV2Literal
69+
*/
70+
private static boolean canLosslessConvertToDateV2Literal(DateTimeV2Literal literal) {
71+
return (literal.getHour() | literal.getMinute() | literal.getSecond()
72+
| literal.getMicroSecond()) == 0L;
73+
}
74+
75+
private DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) {
76+
return new DateV2Literal(literal.getYear(), literal.getMonth(), literal.getDay());
77+
}
78+
}

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.nereids.types.BigIntType;
3030
import org.apache.doris.nereids.types.BooleanType;
3131
import org.apache.doris.nereids.types.DataType;
32+
import org.apache.doris.nereids.types.DateV2Type;
3233
import org.apache.doris.nereids.types.DoubleType;
3334
import org.apache.doris.nereids.types.IntegerType;
3435
import org.apache.doris.nereids.types.StringType;
@@ -85,7 +86,7 @@ protected void assertRewriteAfterTypeCoercion(String expression, String expected
8586
Assertions.assertEquals(expectedExpression.toSql(), rewrittenExpression.toSql());
8687
}
8788

88-
private Expression replaceUnboundSlot(Expression expression, Map<String, Slot> mem) {
89+
protected Expression replaceUnboundSlot(Expression expression, Map<String, Slot> mem) {
8990
List<Expression> children = Lists.newArrayList();
9091
boolean hasNewChildren = false;
9192
for (Expression child : expression.children()) {
@@ -103,7 +104,7 @@ private Expression replaceUnboundSlot(Expression expression, Map<String, Slot> m
103104
return hasNewChildren ? expression.withChildren(children) : expression;
104105
}
105106

106-
private Expression typeCoercion(Expression expression) {
107+
protected Expression typeCoercion(Expression expression) {
107108
return FunctionBinder.INSTANCE.rewrite(expression, null);
108109
}
109110

@@ -121,6 +122,8 @@ private DataType getType(char t) {
121122
return VarcharType.SYSTEM_DEFAULT;
122123
case 'B':
123124
return BooleanType.INSTANCE;
125+
case 'C':
126+
return DateV2Type.INSTANCE;
124127
default:
125128
return BigIntType.INSTANCE;
126129
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.expression;
19+
20+
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
21+
import org.apache.doris.nereids.rules.expression.rules.SimplifyInPredicate;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
25+
import com.google.common.collect.ImmutableList;
26+
import com.google.common.collect.Maps;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
30+
import java.util.Map;
31+
32+
public class SimplifyInPredicateTest extends ExpressionRewriteTestHelper {
33+
34+
@Test
35+
public void test() {
36+
executor = new ExpressionRuleExecutor(ImmutableList.of(
37+
FoldConstantRule.INSTANCE,
38+
SimplifyInPredicate.INSTANCE
39+
));
40+
Map<String, Slot> mem = Maps.newHashMap();
41+
Expression rewrittenExpression = PARSER.parseExpression("cast(CA as DATETIME) in ('1992-01-31 00:00:00', '1992-02-01 00:00:00')");
42+
rewrittenExpression = typeCoercion(replaceUnboundSlot(rewrittenExpression, mem));
43+
rewrittenExpression = executor.rewrite(rewrittenExpression, context);
44+
Expression expectedExpression = PARSER.parseExpression("CA in (cast('1992-01-31' as date), cast('1992-02-01' as date))");
45+
expectedExpression = replaceUnboundSlot(expectedExpression, mem);
46+
executor = new ExpressionRuleExecutor(ImmutableList.of(
47+
FoldConstantRule.INSTANCE
48+
));
49+
expectedExpression = executor.rewrite(expectedExpression, context);
50+
Assertions.assertEquals(expectedExpression.toSql(), rewrittenExpression.toSql());
51+
}
52+
53+
}

regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query83.out

+3-3
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ PhysicalResultSink
3333
--------------------------------PhysicalOlapScan[date_dim]
3434
------------------------------PhysicalDistribute
3535
--------------------------------PhysicalProject
36-
----------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00))
36+
----------------------------------filter(d_date IN (2001-06-06, 2001-09-02, 2001-11-11))
3737
------------------------------------PhysicalOlapScan[date_dim]
3838
------------hashJoin[INNER_JOIN](sr_items.item_id = wr_items.item_id)
3939
--------------PhysicalProject
@@ -61,7 +61,7 @@ PhysicalResultSink
6161
----------------------------------PhysicalOlapScan[date_dim]
6262
--------------------------------PhysicalDistribute
6363
----------------------------------PhysicalProject
64-
------------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00))
64+
------------------------------------filter(d_date IN (2001-06-06, 2001-09-02, 2001-11-11))
6565
--------------------------------------PhysicalOlapScan[date_dim]
6666
--------------PhysicalProject
6767
----------------hashAgg[GLOBAL]
@@ -90,6 +90,6 @@ PhysicalResultSink
9090
----------------------------------PhysicalOlapScan[date_dim]
9191
--------------------------------PhysicalDistribute
9292
----------------------------------PhysicalProject
93-
------------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00))
93+
------------------------------------filter(d_date IN (2001-06-06, 2001-09-02, 2001-11-11))
9494
--------------------------------------PhysicalOlapScan[date_dim]
9595

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_simplify_in_predicate") {
19+
sql "set enable_nereids_planner=true"
20+
sql 'set enable_fallback_to_original_planner=false;'
21+
sql 'drop table if exists test_simplify_in_predicate_t'
22+
sql """CREATE TABLE IF NOT EXISTS `test_simplify_in_predicate_t` (
23+
a DATE NOT NULL
24+
) ENGINE=OLAP
25+
UNIQUE KEY (`a`)
26+
DISTRIBUTED BY HASH(`a`) BUCKETS 120
27+
PROPERTIES (
28+
"replication_num" = "1",
29+
"in_memory" = "false",
30+
"compression" = "LZ4"
31+
);"""
32+
sql """insert into test_simplify_in_predicate_t values( "2023-06-06" );"""
33+
34+
explain {
35+
sql "verbose select * from test_simplify_in_predicate_t where a in ('1992-01-31', '1992-02-01', '1992-02-02', '1992-02-03', '1992-02-04');"
36+
notContains "CAST"
37+
}
38+
}

0 commit comments

Comments
 (0)