Skip to content

Commit 44e9368

Browse files
seawindeseawinde
authored
Pick some fix from master to 21(#41472) (#40106)(#40173) (#42212)
## Proposed changes pr: #41472 commitId: 2745e04 pr: #40106 commitId: 0fdb1ee pr: #40173 commitId: 0d07e3d
1 parent 0d76c16 commit 44e9368

File tree

18 files changed

+910
-9
lines changed

18 files changed

+910
-9
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java

+5
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ public MTMV() {
111111
mvRwLock = new ReentrantReadWriteLock(true);
112112
}
113113

114+
@Override
115+
public boolean needReadLockWhenPlan() {
116+
return true;
117+
}
118+
114119
public MTMVRefreshInfo getRefreshInfo() {
115120
readMvLock();
116121
try {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java

+5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
3131
import org.apache.doris.nereids.parser.NereidsParser;
3232
import org.apache.doris.nereids.properties.PhysicalProperties;
33+
import org.apache.doris.nereids.rules.RuleType;
3334
import org.apache.doris.nereids.trees.plans.Plan;
3435
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3536
import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
@@ -39,6 +40,7 @@
3940
import org.apache.doris.qe.ConnectContext;
4041
import org.apache.doris.qe.SessionVariable;
4142

43+
import com.google.common.collect.ImmutableSet;
4244
import com.google.common.collect.Sets;
4345

4446
import java.util.List;
@@ -57,6 +59,9 @@ public static ConnectContext createMTMVContext(MTMV mtmv) {
5759
ctx.getSessionVariable().enableFallbackToOriginalPlanner = false;
5860
ctx.getSessionVariable().enableNereidsDML = true;
5961
ctx.getSessionVariable().allowModifyMaterializedViewData = true;
62+
// Disable add default limit rule to avoid refresh data wrong
63+
ctx.getSessionVariable().setDisableNereidsRules(
64+
String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name())));
6065
Optional<String> workloadGroup = mtmv.getWorkloadGroup();
6166
if (workloadGroup.isPresent()) {
6267
ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

+9
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ public class CascadesContext implements ScheduleContext {
124124
private final Optional<CascadesContext> parent;
125125

126126
private final Set<MaterializationContext> materializationContexts;
127+
private final Set<List<String>> materializationRewrittenSuccessSet = new HashSet<>();
127128
private boolean isLeadingJoin = false;
128129

129130
private boolean isLeadingDisableJoinReorder = false;
@@ -370,6 +371,14 @@ public void addMaterializationContext(MaterializationContext materializationCont
370371
this.materializationContexts.add(materializationContext);
371372
}
372373

374+
public Set<List<String>> getMaterializationRewrittenSuccessSet() {
375+
return materializationRewrittenSuccessSet;
376+
}
377+
378+
public void addMaterializationRewrittenSuccess(List<String> materializationQualifier) {
379+
this.materializationRewrittenSuccessSet.add(materializationQualifier);
380+
}
381+
373382
/**
374383
* getAndCacheSessionVariable
375384
*/

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,8 @@ public synchronized void addTableReadLock(TableIf tableIf) {
438438
String fullTableName = tableIf.getNameWithFullQualifiers();
439439
String resourceName = "tableReadLock(" + fullTableName + ")";
440440
plannerResources.push(new CloseableResource(
441-
resourceName, Thread.currentThread().getName(), originStatement.originStmt, tableIf::readUnlock));
441+
resourceName, Thread.currentThread().getName(),
442+
originStatement == null ? null : originStatement.originStmt, tableIf::readUnlock));
442443
}
443444

444445
/** releasePlannerResources */

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.nereids.memo;
1919

20+
import org.apache.doris.catalog.TableIf;
2021
import org.apache.doris.common.Pair;
2122
import org.apache.doris.nereids.CascadesContext;
2223
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
@@ -126,6 +127,9 @@ public void refresh(Group group, CascadesContext cascadesContext) {
126127
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
127128
if (groupExpression.children().isEmpty()) {
128129
BitSet leaf = constructLeaf(groupExpression, cascadesContext);
130+
if (leaf.isEmpty()) {
131+
break;
132+
}
129133
groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>()));
130134
continue;
131135
}
@@ -163,9 +167,19 @@ public void refresh(Group group, CascadesContext cascadesContext) {
163167
private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) {
164168
Plan plan = groupExpression.getPlan();
165169
BitSet tableMap = new BitSet();
170+
boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable()
171+
.isEnableMaterializedViewNestRewrite();
166172
if (plan instanceof LogicalCatalogRelation) {
173+
TableIf table = ((LogicalCatalogRelation) plan).getTable();
174+
// If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct
175+
// table id map for nest mv rewrite
176+
if (!enableMaterializedViewNestRewrite
177+
&& cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) {
178+
return tableMap;
179+
180+
}
167181
tableMap.set(cascadesContext.getStatementContext()
168-
.getTableId(((LogicalCatalogRelation) plan).getTable()).asInt());
182+
.getTableId(table).asInt());
169183
}
170184
// one row relation / CTE consumer
171185
return tableMap;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessor.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,13 @@
2727
*/
2828
public class PlanPostProcessor extends DefaultPlanRewriter<CascadesContext> {
2929

30-
public Plan processRoot(Plan plan, CascadesContext ctx) {
31-
AbstractPhysicalPlan newPlan = (AbstractPhysicalPlan) super.visit(plan, ctx);
30+
@Override
31+
public Plan visit(Plan plan, CascadesContext context) {
32+
AbstractPhysicalPlan newPlan = (AbstractPhysicalPlan) super.visit(plan, context);
3233
return newPlan == plan ? plan : newPlan.copyStatsAndGroupIdFrom((AbstractPhysicalPlan) plan);
3334
}
35+
36+
public Plan processRoot(Plan plan, CascadesContext ctx) {
37+
return plan.accept(this, ctx);
38+
}
3439
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

+17-2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
3838
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
3939
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
40+
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
4041
import org.apache.doris.nereids.trees.expressions.Alias;
4142
import org.apache.doris.nereids.trees.expressions.Expression;
4243
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -256,6 +257,12 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
256257
// Rewrite query by view
257258
rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping,
258259
rewrittenPlan, materializationContext, cascadesContext);
260+
// If rewrite successfully, try to get mv read lock to avoid data inconsistent,
261+
// try to get lock which should added before RBO
262+
if (materializationContext instanceof AsyncMaterializationContext && !materializationContext.isSuccess()) {
263+
cascadesContext.getStatementContext()
264+
.addTableReadLock(((AsyncMaterializationContext) materializationContext).getMtmv());
265+
}
259266
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
260267
childContext -> {
261268
Rewriter.getWholeTreeRewriter(childContext).execute();
@@ -349,6 +356,13 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
349356
rewrittenPlanOutput, queryPlan.getOutput()));
350357
continue;
351358
}
359+
// Merge project
360+
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
361+
childContext -> {
362+
Rewriter.getCteChildrenRewriter(childContext,
363+
ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute();
364+
return childContext.getRewritePlan();
365+
}, rewrittenPlan, queryPlan);
352366
if (!isOutputValid(queryPlan, rewrittenPlan)) {
353367
LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties();
354368
materializationContext.recordFailReason(queryStructInfo,
@@ -358,11 +372,11 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
358372
logicalProperties, queryPlan.getLogicalProperties()));
359373
continue;
360374
}
361-
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext);
362375
trySetStatistics(materializationContext, cascadesContext);
363376
rewriteResults.add(rewrittenPlan);
364377
// if rewrite successfully, try to regenerate mv scan because it maybe used again
365378
materializationContext.tryReGenerateScanPlan(cascadesContext);
379+
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext);
366380
}
367381
return rewriteResults;
368382
}
@@ -830,8 +844,9 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon
830844
return checkQueryPattern(structInfo, cascadesContext);
831845
}
832846

833-
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
847+
protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) {
834848
context.setSuccess(true);
849+
cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier());
835850
if (plan.getGroupExpression().isPresent()) {
836851
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true);
837852
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
/**
3333
* This is responsible for join pattern such as project on filter on join
34+
* Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy
3435
*/
3536
public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule {
3637

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java

+14-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.nereids.CascadesContext;
3030
import org.apache.doris.nereids.memo.Group;
3131
import org.apache.doris.nereids.memo.StructInfoMap;
32+
import org.apache.doris.nereids.rules.RuleType;
3233
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
3334
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
3435
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -58,6 +59,7 @@
5859
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
5960
import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
6061
import org.apache.doris.nereids.util.ExpressionUtils;
62+
import org.apache.doris.qe.SessionVariable;
6163

6264
import com.google.common.collect.HashMultimap;
6365
import com.google.common.collect.ImmutableList;
@@ -197,8 +199,8 @@ public static List<StructInfo> extractStructInfo(Plan plan, Plan originalPlan, C
197199
structInfosBuilder.add(structInfo);
198200
}
199201
}
200-
return structInfosBuilder.build();
201202
}
203+
return structInfosBuilder.build();
202204
}
203205
// if plan doesn't belong to any group, construct it directly
204206
return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext));
@@ -247,11 +249,22 @@ public static Plan rewriteByRules(
247249
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
248250
cascadesContext.getStatementContext(), rewrittenPlan,
249251
cascadesContext.getCurrentJobContext().getRequiredProperties());
252+
// Tmp old disable rule variable
253+
Set<String> oldDisableRuleNames = rewrittenPlanContext.getStatementContext().getConnectContext()
254+
.getSessionVariable()
255+
.getDisableNereidsRuleNames();
256+
rewrittenPlanContext.getStatementContext().getConnectContext().getSessionVariable()
257+
.setDisableNereidsRules(String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name())));
258+
rewrittenPlanContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
250259
try {
251260
rewrittenPlanContext.getConnectContext().setSkipAuth(true);
252261
rewrittenPlan = planRewriter.apply(rewrittenPlanContext);
253262
} finally {
254263
rewrittenPlanContext.getConnectContext().setSkipAuth(false);
264+
// Recover old disable rules variable
265+
rewrittenPlanContext.getStatementContext().getConnectContext().getSessionVariable()
266+
.setDisableNereidsRules(String.join(",", oldDisableRuleNames));
267+
rewrittenPlanContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
255268
}
256269
Map<ExprId, Slot> exprIdToNewRewrittenSlot = Maps.newLinkedHashMap();
257270
for (Slot slot : rewrittenPlan.getOutput()) {

‎fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,12 @@ public PlanChecker rewrite() {
260260
public PlanChecker optimize() {
261261
cascadesContext.setJobContext(PhysicalProperties.GATHER);
262262
double now = System.currentTimeMillis();
263-
new Optimizer(cascadesContext).execute();
263+
try {
264+
new Optimizer(cascadesContext).execute();
265+
} finally {
266+
// Mv rewrite add lock manually, so need release manually
267+
cascadesContext.getStatementContext().releasePlannerResources();
268+
}
264269
System.out.println("cascades:" + (System.currentTimeMillis() - now));
265270
return this;
266271
}

‎regression-test/data/mtmv_p0/limit/refresh_with_sql_limit.out

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !query_mv_1 --
3+
1 1 o 10.50 2023-12-08 a b 1 yy
4+
1 1 o 10.50 2023-12-08 a b 1 yy
5+
1 1 o 10.50 2023-12-08 a b 1 yy
6+
1 1 o 9.50 2023-12-08 a b 1 yy
7+
2 1 o 11.50 2023-12-09 a b 1 yy
8+
2 1 o 11.50 2023-12-09 a b 1 yy
9+
2 1 o 11.50 2023-12-09 a b 1 yy
10+
3 1 o 12.50 2023-12-10 a b 1 yy
11+
3 1 o 12.50 2023-12-10 a b 1 yy
12+
3 1 o 12.50 2023-12-10 a b 1 yy
13+
3 1 o 33.50 2023-12-10 a b 1 yy
14+
4 2 o 43.20 2023-12-11 c d 2 mm
15+
4 2 o 43.20 2023-12-11 c d 2 mm
16+
4 2 o 43.20 2023-12-11 c d 2 mm
17+
5 2 o 1.20 2023-12-12 c d 2 mi
18+
5 2 o 56.20 2023-12-12 c d 2 mi
19+
5 2 o 56.20 2023-12-12 c d 2 mi
20+
5 2 o 56.20 2023-12-12 c d 2 mi
21+
22+
-- !query_mv_2 --
23+
1 1 o 10.50 2023-12-08 a b 1 yy
24+
1 1 o 10.50 2023-12-08 a b 1 yy
25+
1 1 o 10.50 2023-12-08 a b 1 yy
26+
1 1 o 9.50 2023-12-08 a b 1 yy
27+
2 1 o 11.50 2023-12-09 a b 1 yy
28+
2 1 o 11.50 2023-12-09 a b 1 yy
29+
2 1 o 11.50 2023-12-09 a b 1 yy
30+
3 1 o 12.50 2023-12-10 a b 1 yy
31+
3 1 o 12.50 2023-12-10 a b 1 yy
32+
3 1 o 12.50 2023-12-10 a b 1 yy
33+
3 1 o 33.50 2023-12-10 a b 1 yy
34+
4 2 o 43.20 2023-12-11 c d 2 mm
35+
4 2 o 43.20 2023-12-11 c d 2 mm
36+
4 2 o 43.20 2023-12-11 c d 2 mm
37+
5 2 o 1.20 2023-12-12 c d 2 mi
38+
5 2 o 56.20 2023-12-12 c d 2 mi
39+
5 2 o 56.20 2023-12-12 c d 2 mi
40+
5 2 o 56.20 2023-12-12 c d 2 mi
41+

‎regression-test/data/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.out

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !query1_0_before --
3+
4
4+
5+
-- !query1_0_after --
6+
4
7+
8+
-- !query2_0_before --
9+
4
10+
11+
-- !query2_0_after --
12+
4
13+
14+
-- !query3_0_before --
15+
4
16+
17+
-- !query3_0_after --
18+
4
19+

0 commit comments

Comments
 (0)