From 8d5d500a435029869eec072712a6a64301feff5e Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 2 Sep 2024 10:01:43 +0800 Subject: [PATCH] disable limit in mv refresh --- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 5 + .../mtmv_p0/limit/refresh_with_sql_limit.out | 41 +++++++ .../limit/refresh_with_sql_limit.groovy | 115 ++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 regression-test/data/mtmv_p0/limit/refresh_with_sql_limit.out create mode 100644 regression-test/suites/mtmv_p0/limit/refresh_with_sql_limit.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 8fb56f5ddac0341..0fa68707771e6ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; @@ -39,6 +40,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import java.util.List; @@ -56,6 +58,9 @@ public static ConnectContext createMTMVContext(MTMV mtmv) { ctx.setThreadLocalInfo(); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().allowModifyMaterializedViewData = true; + // Disable add default limit rule to avoid refresh data wrong + ctx.getSessionVariable().setDisableNereidsRules( + String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name()))); Optional workloadGroup = mtmv.getWorkloadGroup(); if (workloadGroup.isPresent()) { ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get()); diff --git a/regression-test/data/mtmv_p0/limit/refresh_with_sql_limit.out b/regression-test/data/mtmv_p0/limit/refresh_with_sql_limit.out new file mode 100644 index 000000000000000..e2dee09f17451d5 --- /dev/null +++ b/regression-test/data/mtmv_p0/limit/refresh_with_sql_limit.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_mv_1 -- +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 9.50 2023-12-08 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 33.50 2023-12-10 a b 1 yy +4 2 o 43.20 2023-12-11 c d 2 mm +4 2 o 43.20 2023-12-11 c d 2 mm +4 2 o 43.20 2023-12-11 c d 2 mm +5 2 o 1.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi + + +-- !query_mv_2 -- +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 10.50 2023-12-08 a b 1 yy +1 1 o 9.50 2023-12-08 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +2 1 o 11.50 2023-12-09 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 12.50 2023-12-10 a b 1 yy +3 1 o 33.50 2023-12-10 a b 1 yy +4 2 o 43.20 2023-12-11 c d 2 mm +4 2 o 43.20 2023-12-11 c d 2 mm +4 2 o 43.20 2023-12-11 c d 2 mm +5 2 o 1.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi +5 2 o 56.20 2023-12-12 c d 2 mi \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/limit/refresh_with_sql_limit.groovy b/regression-test/suites/mtmv_p0/limit/refresh_with_sql_limit.groovy new file mode 100644 index 000000000000000..2b8ac5f8467ef16 --- /dev/null +++ b/regression-test/suites/mtmv_p0/limit/refresh_with_sql_limit.groovy @@ -0,0 +1,115 @@ +package limit +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("refresh_with_sql_limit") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + sql """analyze table orders with sync""" + + + sql """DROP MATERIALIZED VIEW IF EXISTS mv_1""" + sql """set global default_order_by_limit = 2;""" + sql """set global sql_select_limit = 2;""" + sql""" + CREATE MATERIALIZED VIEW mv_1 + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders; + """ + sql """refresh materialized view mv_1 auto;""" + def job_name = getJobName(db, "mv_1"); + waitingMTMVTaskFinished(job_name) + + // Reset and test mv data is right or not + sql """set global default_order_by_limit = -1;""" + sql """set global sql_select_limit = -1;""" + order_qt_query_mv_1 "select * from mv_1" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_1""" + + + sql """DROP MATERIALIZED VIEW IF EXISTS mv_2""" + sql """set global default_order_by_limit = 2""" + sql """set global sql_select_limit = 2""" + sql""" + CREATE MATERIALIZED VIEW mv_2 + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders; + """ + waitingMTMVTaskFinished(getJobName(db, "mv_2")) + + // Reset and test mv data is right or not + sql """set global default_order_by_limit = -1;""" + sql """set global sql_select_limit = -1;""" + order_qt_query_mv_2 "select * from mv_2" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_2""" +}