Skip to content

Commit 2745e04

Browse files
authored
[fix](mtmv) Fix enable_materialized_view_nest_rewrite session variable is useless in some scene (#41472)
This is brought by #34050 if set `enable_materialized_view_nest_rewrite = false`, as expected, top level materialized view should rewritten fail, but now successfully. Such as first level materialized view def is CREATE MATERIALIZED VIEW level1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey; second level materialized view def is CREATE MATERIALIZED VIEW level2 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, o_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM join_mv1 GROUP BY l_orderkey, l_linenumber, o_orderkey; if set `enable_materialized_view_nest_rewrite = false`, only `level1` can rewriten succesfully and chosen by cbo if set `enable_materialized_view_nest_rewrite = true`, both `level1` and `level2` can rewriten succesfully and `level2` should be chosen by cbo. This pr fixed this
1 parent 0739991 commit 2745e04

File tree

9 files changed

+207
-51
lines changed

9 files changed

+207
-51
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

+9
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ public class CascadesContext implements ScheduleContext {
122122
private final Optional<CascadesContext> parent;
123123

124124
private final Set<MaterializationContext> materializationContexts;
125+
private final Set<List<String>> materializationRewrittenSuccessSet = new HashSet<>();
125126
private boolean isLeadingJoin = false;
126127

127128
private boolean isLeadingDisableJoinReorder = false;
@@ -366,6 +367,14 @@ public void addMaterializationContext(MaterializationContext materializationCont
366367
this.materializationContexts.add(materializationContext);
367368
}
368369

370+
public Set<List<String>> getMaterializationRewrittenSuccessSet() {
371+
return materializationRewrittenSuccessSet;
372+
}
373+
374+
public void addMaterializationRewrittenSuccess(List<String> materializationQualifier) {
375+
this.materializationRewrittenSuccessSet.add(materializationQualifier);
376+
}
377+
369378
/**
370379
* getAndCacheSessionVariable
371380
*/

fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.nereids.memo;
1919

20+
import org.apache.doris.catalog.TableIf;
2021
import org.apache.doris.common.Pair;
2122
import org.apache.doris.nereids.CascadesContext;
2223
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
@@ -126,6 +127,9 @@ public void refresh(Group group, CascadesContext cascadesContext) {
126127
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
127128
if (groupExpression.children().isEmpty()) {
128129
BitSet leaf = constructLeaf(groupExpression, cascadesContext);
130+
if (leaf.isEmpty()) {
131+
break;
132+
}
129133
groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>()));
130134
continue;
131135
}
@@ -163,9 +167,19 @@ public void refresh(Group group, CascadesContext cascadesContext) {
163167
private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) {
164168
Plan plan = groupExpression.getPlan();
165169
BitSet tableMap = new BitSet();
170+
boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable()
171+
.isEnableMaterializedViewNestRewrite();
166172
if (plan instanceof LogicalCatalogRelation) {
173+
TableIf table = ((LogicalCatalogRelation) plan).getTable();
174+
// If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct
175+
// table id map for nest mv rewrite
176+
if (!enableMaterializedViewNestRewrite
177+
&& cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) {
178+
return tableMap;
179+
180+
}
167181
tableMap.set(cascadesContext.getStatementContext()
168-
.getTableId(((LogicalCatalogRelation) plan).getTable()).asInt());
182+
.getTableId(table).asInt());
169183
}
170184
// one row relation / CTE consumer
171185
return tableMap;

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java

-2
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule;
4040
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule;
4141
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule;
42-
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule;
4342
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyScanRule;
4443
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule;
4544
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule;
@@ -237,7 +236,6 @@ public class RuleSet {
237236
.build();
238237

239238
public static final List<Rule> MATERIALIZED_VIEW_RULES = planRuleFactories()
240-
.add(MaterializedViewOnlyJoinRule.INSTANCE)
241239
.add(MaterializedViewProjectJoinRule.INSTANCE)
242240
.add(MaterializedViewFilterJoinRule.INSTANCE)
243241
.add(MaterializedViewFilterProjectJoinRule.INSTANCE)

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
3838
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
3939
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
40+
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
4041
import org.apache.doris.nereids.trees.expressions.Alias;
4142
import org.apache.doris.nereids.trees.expressions.Expression;
4243
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -354,6 +355,13 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
354355
rewrittenPlanOutput, queryPlan.getOutput()));
355356
continue;
356357
}
358+
// Merge project
359+
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
360+
childContext -> {
361+
Rewriter.getCteChildrenRewriter(childContext,
362+
ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute();
363+
return childContext.getRewritePlan();
364+
}, rewrittenPlan, queryPlan);
357365
if (!isOutputValid(queryPlan, rewrittenPlan)) {
358366
LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties();
359367
materializationContext.recordFailReason(queryStructInfo,
@@ -363,7 +371,7 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
363371
logicalProperties, queryPlan.getLogicalProperties()));
364372
continue;
365373
}
366-
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext);
374+
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext);
367375
trySetStatistics(materializationContext, cascadesContext);
368376
rewriteResults.add(rewrittenPlan);
369377
// if rewrite successfully, try to regenerate mv scan because it maybe used again
@@ -852,8 +860,9 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon
852860
return checkQueryPattern(structInfo, cascadesContext);
853861
}
854862

855-
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
863+
protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) {
856864
context.setSuccess(true);
865+
cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier());
857866
if (plan.getGroupExpression().isPresent()) {
858867
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true);
859868
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java

-45
This file was deleted.

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
/**
3333
* This is responsible for join pattern such as project on filter on join
34+
* Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy
3435
*/
3536
public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule {
3637

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -212,8 +212,8 @@ public static List<StructInfo> extractStructInfo(Plan plan, Plan originalPlan, C
212212
structInfosBuilder.add(structInfo);
213213
}
214214
}
215-
return structInfosBuilder.build();
216215
}
216+
return structInfosBuilder.build();
217217
}
218218
// if plan doesn't belong to any group, construct it directly
219219
return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext));

regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ suite("direct_query_mv") {
2020
String db = context.config.getDbNameByFile(context.file)
2121
sql "use ${db}"
2222
sql "set runtime_filter_mode=OFF"
23+
sql """set enable_materialized_view_nest_rewrite = true; """
2324

2425
sql """
2526
drop table if exists orders
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("nested_mtmv_rewrite_switch") {
19+
String db = context.config.getDbNameByFile(context.file)
20+
sql "use ${db}"
21+
sql "SET enable_materialized_view_rewrite=true"
22+
23+
sql """
24+
drop table if exists orders_2
25+
"""
26+
sql """
27+
CREATE TABLE `orders_2` (
28+
`o_orderkey` BIGINT,
29+
`o_custkey` int,
30+
`o_orderstatus` VARCHAR(1),
31+
`o_totalprice` DECIMAL(15, 2),
32+
`o_orderpriority` VARCHAR(15),
33+
`o_clerk` VARCHAR(15),
34+
`o_shippriority` int,
35+
`o_comment` VARCHAR(79),
36+
`o_orderdate` DATE
37+
) ENGINE=olap
38+
39+
PROPERTIES (
40+
"replication_num" = "1"
41+
42+
);
43+
"""
44+
45+
sql """
46+
drop table if exists lineitem_2
47+
"""
48+
sql """
49+
CREATE TABLE `lineitem_2` (
50+
`l_orderkey` BIGINT,
51+
`l_linenumber` INT,
52+
`l_partkey` INT,
53+
`l_suppkey` INT,
54+
`l_quantity` DECIMAL(15, 2),
55+
`l_extendedprice` DECIMAL(15, 2),
56+
`l_discount` DECIMAL(15, 2),
57+
`l_tax` DECIMAL(15, 2),
58+
`l_returnflag` VARCHAR(1),
59+
`l_linestatus` VARCHAR(1),
60+
`l_commitdate` DATE,
61+
`l_receiptdate` DATE,
62+
`l_shipinstruct` VARCHAR(25),
63+
`l_shipmode` VARCHAR(10),
64+
`l_comment` VARCHAR(44),
65+
`l_shipdate` DATE
66+
) ENGINE=olap
67+
68+
PROPERTIES (
69+
"replication_num" = "1"
70+
71+
);
72+
"""
73+
74+
sql """
75+
insert into orders_2 values
76+
(null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
77+
(1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
78+
(3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
79+
(1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
80+
(2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
81+
(3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
82+
(1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
83+
(2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
84+
(3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
85+
(4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19');
86+
"""
87+
88+
sql"""
89+
insert into lineitem_2 values
90+
(null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
91+
(1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
92+
(3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
93+
(1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
94+
(2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
95+
(3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
96+
(1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
97+
"""
98+
99+
sql """analyze table orders_2 with sync;"""
100+
sql """analyze table lineitem_2 with sync;"""
101+
102+
103+
def compare_res = { def stmt ->
104+
sql "SET enable_materialized_view_rewrite=false"
105+
def origin_res = sql stmt
106+
logger.info("origin_res: " + origin_res)
107+
sql "SET enable_materialized_view_rewrite=true"
108+
def mv_origin_res = sql stmt
109+
logger.info("mv_origin_res: " + mv_origin_res)
110+
assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
111+
for (int row = 0; row < mv_origin_res.size(); row++) {
112+
assertTrue(mv_origin_res[row].size() == origin_res[row].size())
113+
for (int col = 0; col < mv_origin_res[row].size(); col++) {
114+
assertTrue(mv_origin_res[row][col] == origin_res[row][col])
115+
}
116+
}
117+
}
118+
119+
120+
// create base first level mv
121+
create_async_mv(db, "join_mv1", """
122+
SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey
123+
FROM lineitem_2 INNER JOIN orders_2
124+
ON l_orderkey = o_orderkey;
125+
""")
126+
127+
// create second level mv based on first level mv
128+
create_async_mv(db, "agg_mv2", """
129+
SELECT
130+
l_orderkey,
131+
l_linenumber,
132+
o_orderkey,
133+
sum(l_partkey) AS total_revenue,
134+
max(o_custkey) AS max_discount
135+
FROM join_mv1
136+
GROUP BY l_orderkey, l_linenumber, o_orderkey;
137+
""")
138+
139+
// create third level mv based on second level mv
140+
create_async_mv(db, "join_agg_mv3", """
141+
SELECT
142+
l_orderkey,
143+
sum(total_revenue) AS total_revenue,
144+
max(max_discount) AS max_discount
145+
FROM agg_mv2
146+
GROUP BY l_orderkey;
147+
""")
148+
149+
def query = """
150+
SELECT l_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey GROUP BY l_orderkey
151+
"""
152+
153+
sql """set enable_materialized_view_nest_rewrite = false;"""
154+
// Just first level mv rewrite successfully, second and third level mv should rewriten fail
155+
mv_rewrite_fail(query, "agg_mv2")
156+
mv_rewrite_fail(query, "join_agg_mv3")
157+
mv_rewrite_success(query, "join_mv1")
158+
compare_res(query + " order by 1,2,3")
159+
160+
161+
sql """set enable_materialized_view_nest_rewrite = true;"""
162+
// All mv rewrite successfully but only thirst level mv can be chosen by cbo
163+
mv_rewrite_success_without_check_chosen(query, "join_mv1")
164+
mv_rewrite_success_without_check_chosen(query, "agg_mv2")
165+
mv_rewrite_success(query, "join_agg_mv3")
166+
compare_res(query + " order by 1,2,3")
167+
168+
169+
}

0 commit comments

Comments
 (0)