Skip to content

Commit a1690f0

Browse files
seawindeseawinde
authored
[pick](mtmv) pick 4 PR from master #41472 #40106 #40173 #42206 (#42325)
pick from master pr: #41472 commitId: 2745e04 pr: #40106 commitId: 0fdb1ee pr: #40173 commitId: 0d07e3d pr: #42206 commitId: 2bcaa5b
1 parent d27cab0 commit a1690f0

File tree

24 files changed

+980
-97
lines changed

24 files changed

+980
-97
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java

+5
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@ public MTMV() {
113113
mvRwLock = new ReentrantReadWriteLock(true);
114114
}
115115

116+
@Override
117+
public boolean needReadLockWhenPlan() {
118+
return true;
119+
}
120+
116121
public MTMVRefreshInfo getRefreshInfo() {
117122
readMvLock();
118123
try {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java

+5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
3131
import org.apache.doris.nereids.parser.NereidsParser;
3232
import org.apache.doris.nereids.properties.PhysicalProperties;
33+
import org.apache.doris.nereids.rules.RuleType;
3334
import org.apache.doris.nereids.trees.plans.Plan;
3435
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3536
import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
@@ -39,6 +40,7 @@
3940
import org.apache.doris.qe.ConnectContext;
4041
import org.apache.doris.qe.SessionVariable;
4142

43+
import com.google.common.collect.ImmutableSet;
4244
import com.google.common.collect.Sets;
4345

4446
import java.util.List;
@@ -55,6 +57,9 @@ public static ConnectContext createMTMVContext(MTMV mtmv) {
5557
ctx.getState().reset();
5658
ctx.setThreadLocalInfo();
5759
ctx.getSessionVariable().allowModifyMaterializedViewData = true;
60+
// Disable add default limit rule to avoid refresh data wrong
61+
ctx.getSessionVariable().setDisableNereidsRules(
62+
String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name())));
5863
Optional<String> workloadGroup = mtmv.getWorkloadGroup();
5964
if (workloadGroup.isPresent()) {
6065
ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

+9
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ public class CascadesContext implements ScheduleContext {
122122
private final Optional<CascadesContext> parent;
123123

124124
private final Set<MaterializationContext> materializationContexts;
125+
private final Set<List<String>> materializationRewrittenSuccessSet = new HashSet<>();
125126
private boolean isLeadingJoin = false;
126127

127128
private boolean isLeadingDisableJoinReorder = false;
@@ -368,6 +369,14 @@ public void addMaterializationContext(MaterializationContext materializationCont
368369
this.materializationContexts.add(materializationContext);
369370
}
370371

372+
public Set<List<String>> getMaterializationRewrittenSuccessSet() {
373+
return materializationRewrittenSuccessSet;
374+
}
375+
376+
public void addMaterializationRewrittenSuccess(List<String> materializationQualifier) {
377+
this.materializationRewrittenSuccessSet.add(materializationQualifier);
378+
}
379+
371380
/**
372381
* getAndCacheSessionVariable
373382
*/

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,8 @@ public synchronized void addTableReadLock(TableIf tableIf) {
441441
String fullTableName = tableIf.getNameWithFullQualifiers();
442442
String resourceName = "tableReadLock(" + fullTableName + ")";
443443
plannerResources.push(new CloseableResource(
444-
resourceName, Thread.currentThread().getName(), originStatement.originStmt, tableIf::readUnlock));
444+
resourceName, Thread.currentThread().getName(),
445+
originStatement == null ? null : originStatement.originStmt, tableIf::readUnlock));
445446
}
446447

447448
/** releasePlannerResources */

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.nereids.memo;
1919

20+
import org.apache.doris.catalog.TableIf;
2021
import org.apache.doris.common.Pair;
2122
import org.apache.doris.nereids.CascadesContext;
2223
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
@@ -126,6 +127,9 @@ public void refresh(Group group, CascadesContext cascadesContext) {
126127
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
127128
if (groupExpression.children().isEmpty()) {
128129
BitSet leaf = constructLeaf(groupExpression, cascadesContext);
130+
if (leaf.isEmpty()) {
131+
break;
132+
}
129133
groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>()));
130134
continue;
131135
}
@@ -163,9 +167,19 @@ public void refresh(Group group, CascadesContext cascadesContext) {
163167
private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) {
164168
Plan plan = groupExpression.getPlan();
165169
BitSet tableMap = new BitSet();
170+
boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable()
171+
.isEnableMaterializedViewNestRewrite();
166172
if (plan instanceof LogicalCatalogRelation) {
173+
TableIf table = ((LogicalCatalogRelation) plan).getTable();
174+
// If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct
175+
// table id map for nest mv rewrite
176+
if (!enableMaterializedViewNestRewrite
177+
&& cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) {
178+
return tableMap;
179+
180+
}
167181
tableMap.set(cascadesContext.getStatementContext()
168-
.getTableId(((LogicalCatalogRelation) plan).getTable()).asInt());
182+
.getTableId(table).asInt());
169183
}
170184
// one row relation / CTE consumer
171185
return tableMap;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java

-2
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule;
4040
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule;
4141
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule;
42-
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule;
4342
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyScanRule;
4443
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule;
4544
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule;
@@ -228,7 +227,6 @@ public class RuleSet {
228227
.build();
229228

230229
public static final List<Rule> MATERIALIZED_VIEW_RULES = planRuleFactories()
231-
.add(MaterializedViewOnlyJoinRule.INSTANCE)
232230
.add(MaterializedViewProjectJoinRule.INSTANCE)
233231
.add(MaterializedViewFilterJoinRule.INSTANCE)
234232
.add(MaterializedViewFilterProjectJoinRule.INSTANCE)

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

+17-2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
3838
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
3939
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
40+
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
4041
import org.apache.doris.nereids.trees.expressions.Alias;
4142
import org.apache.doris.nereids.trees.expressions.Expression;
4243
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -261,6 +262,12 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
261262
// Rewrite query by view
262263
rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping,
263264
rewrittenPlan, materializationContext, cascadesContext);
265+
// If rewrite successfully, try to get mv read lock to avoid data inconsistent,
266+
// try to get lock which should added before RBO
267+
if (materializationContext instanceof AsyncMaterializationContext && !materializationContext.isSuccess()) {
268+
cascadesContext.getStatementContext()
269+
.addTableReadLock(((AsyncMaterializationContext) materializationContext).getMtmv());
270+
}
264271
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
265272
childContext -> {
266273
Rewriter.getWholeTreeRewriter(childContext).execute();
@@ -354,6 +361,13 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
354361
rewrittenPlanOutput, queryPlan.getOutput()));
355362
continue;
356363
}
364+
// Merge project
365+
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
366+
childContext -> {
367+
Rewriter.getCteChildrenRewriter(childContext,
368+
ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute();
369+
return childContext.getRewritePlan();
370+
}, rewrittenPlan, queryPlan);
357371
if (!isOutputValid(queryPlan, rewrittenPlan)) {
358372
LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties();
359373
materializationContext.recordFailReason(queryStructInfo,
@@ -363,11 +377,11 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
363377
logicalProperties, queryPlan.getLogicalProperties()));
364378
continue;
365379
}
366-
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext);
367380
trySetStatistics(materializationContext, cascadesContext);
368381
rewriteResults.add(rewrittenPlan);
369382
// if rewrite successfully, try to regenerate mv scan because it maybe used again
370383
materializationContext.tryReGenerateScanPlan(cascadesContext);
384+
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext);
371385
}
372386
return rewriteResults;
373387
}
@@ -852,8 +866,9 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon
852866
return checkQueryPattern(structInfo, cascadesContext);
853867
}
854868

855-
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
869+
protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) {
856870
context.setSuccess(true);
871+
cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier());
857872
if (plan.getGroupExpression().isPresent()) {
858873
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true);
859874
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java

-45
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
/**
3333
* This is responsible for join pattern such as project on filter on join
34+
* Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy
3435
*/
3536
public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule {
3637

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import org.apache.doris.nereids.util.ExpressionUtils;
7474
import org.apache.doris.qe.ConnectContext;
7575
import org.apache.doris.qe.OriginStatement;
76+
import org.apache.doris.qe.SessionVariable;
7677

7778
import com.google.common.collect.HashMultimap;
7879
import com.google.common.collect.ImmutableList;
@@ -212,8 +213,8 @@ public static List<StructInfo> extractStructInfo(Plan plan, Plan originalPlan, C
212213
structInfosBuilder.add(structInfo);
213214
}
214215
}
215-
return structInfosBuilder.build();
216216
}
217+
return structInfosBuilder.build();
217218
}
218219
// if plan doesn't belong to any group, construct it directly
219220
return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext));
@@ -267,11 +268,22 @@ public static Plan rewriteByRules(
267268
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
268269
cascadesContext.getStatementContext(), rewrittenPlan,
269270
cascadesContext.getCurrentJobContext().getRequiredProperties());
271+
// Tmp old disable rule variable
272+
Set<String> oldDisableRuleNames = rewrittenPlanContext.getStatementContext().getConnectContext()
273+
.getSessionVariable()
274+
.getDisableNereidsRuleNames();
275+
rewrittenPlanContext.getStatementContext().getConnectContext().getSessionVariable()
276+
.setDisableNereidsRules(String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name())));
277+
rewrittenPlanContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
270278
try {
271279
rewrittenPlanContext.getConnectContext().setSkipAuth(true);
272280
rewrittenPlan = planRewriter.apply(rewrittenPlanContext);
273281
} finally {
274282
rewrittenPlanContext.getConnectContext().setSkipAuth(false);
283+
// Recover old disable rules variable
284+
rewrittenPlanContext.getStatementContext().getConnectContext().getSessionVariable()
285+
.setDisableNereidsRules(String.join(",", oldDisableRuleNames));
286+
rewrittenPlanContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
275287
}
276288
Map<ExprId, Slot> exprIdToNewRewrittenSlot = Maps.newLinkedHashMap();
277289
for (Slot slot : rewrittenPlan.getOutput()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterViewInfo.java

-19
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import org.apache.doris.analysis.AlterViewStmt;
2121
import org.apache.doris.analysis.ColWithComment;
2222
import org.apache.doris.analysis.TableName;
23-
import org.apache.doris.catalog.Column;
2423
import org.apache.doris.catalog.DatabaseIf;
2524
import org.apache.doris.catalog.Env;
2625
import org.apache.doris.catalog.TableIf;
@@ -31,19 +30,13 @@
3130
import org.apache.doris.common.UserException;
3231
import org.apache.doris.common.util.Util;
3332
import org.apache.doris.mysql.privilege.PrivPredicate;
34-
import org.apache.doris.nereids.NereidsPlanner;
35-
import org.apache.doris.nereids.analyzer.UnboundResultSink;
36-
import org.apache.doris.nereids.properties.PhysicalProperties;
3733
import org.apache.doris.nereids.trees.expressions.Slot;
38-
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3934
import org.apache.doris.nereids.util.PlanUtils;
4035
import org.apache.doris.qe.ConnectContext;
4136

4237
import com.google.common.collect.Lists;
43-
import com.google.common.collect.Sets;
4438

4539
import java.util.List;
46-
import java.util.Set;
4740

4841
/** AlterViewInfo */
4942
public class AlterViewInfo extends BaseViewInfo {
@@ -83,18 +76,6 @@ public void init(ConnectContext ctx) throws UserException {
8376
createFinalCols(outputs);
8477
}
8578

86-
/**validate*/
87-
public void validate(ConnectContext ctx) throws UserException {
88-
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
89-
planner.planWithLock(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE);
90-
Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
91-
for (Column col : finalCols) {
92-
if (!colSets.add(col.getName())) {
93-
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName());
94-
}
95-
}
96-
}
97-
9879
/**translateToLegacyStmt*/
9980
public AlterViewStmt translateToLegacyStmt(ConnectContext ctx) {
10081
List<ColWithComment> cols = Lists.newArrayList();

0 commit comments

Comments
 (0)