Skip to content

Commit 1f9aa8a

Browse files
mymeiyimymeiyi
authored
[fix](group commit) Fix some group commit problems (#27769)
1 parent d96e2df commit 1f9aa8a

13 files changed

+370
-253
lines changed

‎be/src/vec/sink/group_commit_block_sink.cpp

+7
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,13 @@ Status GroupCommitBlockSink::_add_block(RuntimeState* state,
140140
if (block->rows() == 0) {
141141
return Status::OK();
142142
}
143+
for (int i = 0; i < block->columns(); ++i) {
144+
if (block->get_by_position(i).type->is_nullable()) {
145+
continue;
146+
}
147+
block->get_by_position(i).column = make_nullable(block->get_by_position(i).column);
148+
block->get_by_position(i).type = make_nullable(block->get_by_position(i).type);
149+
}
143150
// add block to queue
144151
auto _cur_mutable_block = vectorized::MutableBlock::create_unique(block->clone_empty());
145152
{

‎fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ public void getTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String
253253
OlapTable olapTable = (OlapTable) table;
254254
tblName.setDb(olapTable.getDatabase().getFullName());
255255
tblName.setTbl(olapTable.getName());
256-
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) {
256+
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS || olapTable.getTableProperty().storeRowColumn()) {
257257
List<Column> columns = Lists.newArrayList(olapTable.getBaseSchema(true));
258258
targetColumnNames = columns.stream().map(c -> c.getName()).collect(Collectors.toList());
259259
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -178,12 +178,11 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
178178
}
179179

180180
OlapTableSink sink = ((OlapTableSink) planner.getFragments().get(0).getSink());
181-
if (ctx.getSessionVariable().isEnableInsertGroupCommit()) {
182-
// group commit
183-
if (analyzeGroupCommit(sink, physicalOlapTableSink)) {
184-
handleGroupCommit(ctx, sink, physicalOlapTableSink);
185-
return;
186-
}
181+
// group commit
182+
if (analyzeGroupCommit(sink, physicalOlapTableSink)) {
183+
/*handleGroupCommit(ctx, sink, physicalOlapTableSink);
184+
return;*/
185+
throw new AnalysisException("group commit is not supported in nereids now");
187186
}
188187
Preconditions.checkArgument(!isTxnBegin, "an insert command cannot create more than one txn");
189188
Transaction txn = new Transaction(ctx,

‎regression-test/data/insert_p0/insert_group_commit_into_unique.out

+90
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
14 d 24 0
1313
15 c 23 0
1414
16 d 24 1
15+
27 e 25 0
1516

1617
-- !sql --
1718
2 b -1 0
@@ -23,6 +24,7 @@
2324
13 c 23 0
2425
14 d 24 0
2526
15 c 23 0
27+
27 e 25 0
2628

2729
-- !sql --
2830
1 a 10 10 1
@@ -37,6 +39,7 @@
3739
14 d 24 24 0
3840
15 c 23 23 0
3941
16 d 24 24 1
42+
27 e 25 25 0
4043

4144
-- !sql --
4245
2 b 30 30 0
@@ -48,6 +51,7 @@
4851
13 c 23 23 0
4952
14 d 24 24 0
5053
15 c 23 23 0
54+
27 e 25 25 0
5155

5256
-- !sql --
5357
1 a 200 200 1
@@ -59,6 +63,9 @@
5963
11 a 11 10 1
6064
12 a 12 10 0
6165
13 a 13 10 0
66+
20 b 20 8 0
67+
21 b 21 7 0
68+
22 b 22 6 0
6269

6370
-- !sql --
6471
2 b 30 200 0
@@ -68,4 +75,87 @@
6875
10 a 10 11 0
6976
12 a 12 10 0
7077
13 a 13 10 0
78+
20 b 20 8 0
79+
21 b 21 7 0
80+
22 b 22 6 0
81+
82+
-- !sql --
83+
1 a 10 1
84+
2 b -1 0
85+
3 c -1 0
86+
4 \N -1 0
87+
5 q 50 0
88+
6 \N -1 0
89+
11 a 211 0
90+
12 b 22 1
91+
13 c 23 0
92+
14 d 24 0
93+
15 c 23 0
94+
16 d 24 1
95+
27 e 25 0
96+
97+
-- !sql --
98+
2 b -1 0
99+
3 c -1 0
100+
4 \N -1 0
101+
5 q 50 0
102+
6 \N -1 0
103+
11 a 211 0
104+
13 c 23 0
105+
14 d 24 0
106+
15 c 23 0
107+
27 e 25 0
108+
109+
-- !sql --
110+
1 a 10 10 1
111+
2 b 30 30 0
112+
3 c 30 30 0
113+
4 \N 70 70 0
114+
5 q 50 50 0
115+
6 \N 60 60 0
116+
11 a 211 211 0
117+
12 b 22 22 1
118+
13 c 23 23 0
119+
14 d 24 24 0
120+
15 c 23 23 0
121+
16 d 24 24 1
122+
27 e 25 25 0
123+
124+
-- !sql --
125+
2 b 30 30 0
126+
3 c 30 30 0
127+
4 \N 70 70 0
128+
5 q 50 50 0
129+
6 \N 60 60 0
130+
11 a 211 211 0
131+
13 c 23 23 0
132+
14 d 24 24 0
133+
15 c 23 23 0
134+
27 e 25 25 0
135+
136+
-- !sql --
137+
1 a 200 200 1
138+
2 b 30 200 0
139+
3 c 30 300 0
140+
5 q 50 500 0
141+
6 \N 60 600 0
142+
10 a 10 11 0
143+
11 a 11 10 1
144+
12 a 12 10 0
145+
13 a 13 10 0
146+
20 b 20 8 0
147+
21 b 21 7 0
148+
22 b 22 6 0
149+
150+
-- !sql --
151+
2 b 30 200 0
152+
3 c 30 300 0
153+
5 q 50 500 0
154+
6 \N 60 600 0
155+
10 a 10 11 0
156+
12 a 12 10 0
157+
13 a 13 10 0
158+
20 b 20 8 0
159+
21 b 21 7 0
160+
22 b 22 6 0
71161

‎regression-test/data/insert_p0/insert_with_null.out

+1-2
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,10 @@
4343
4 null []
4444
5 NULL ["k5, k6"]
4545
6 \N ["k7", "k8"]
46-
7 abc \N
46+
7 abc []
4747

4848
-- !sql --
4949
6 \N ["k7", "k8"]
5050

5151
-- !sql --
52-
7 abc \N
5352

‎regression-test/data/insert_p0/test_group_commit_2.csv

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
11,a,211,0
22
12,b,22,1
33
15,c,23,0
4-
16,d,24,1
4+
16,d,24,1
5+
27,e,25,0

‎regression-test/data/insert_p0/test_group_commit_4.csv

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
10,a,10,11,0
22
11,a,11,10,1
33
12,a,12,9,0
4-
13,a,13,9,1
4+
13,a,13,9,1
5+
20,b,20,8,0
6+
21,b,21,7,0
7+
22,b,22,6,0

‎regression-test/suites/insert_p0/insert_group_commit_into.groovy

+6-5
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ suite("insert_group_commit_into") {
7979
assertTrue(serverInfo.contains("'status':'VISIBLE'"))
8080
assertTrue(!serverInfo.contains("'label':'group_commit_"))
8181
}
82+
8283
for (item in ["legacy", "nereids"]) {
8384
try {
8485
// create table
@@ -106,7 +107,7 @@ suite("insert_group_commit_into") {
106107
if (item == "nereids") {
107108
sql """ set enable_nereids_dml = true; """
108109
sql """ set enable_nereids_planner=true; """
109-
sql """ set enable_fallback_to_original_planner=false; """
110+
//sql """ set enable_fallback_to_original_planner=false; """
110111
} else {
111112
sql """ set enable_nereids_dml = false; """
112113
}
@@ -194,16 +195,16 @@ suite("insert_group_commit_into") {
194195
qt_sql """ select name, score from ${table} order by name asc; """
195196

196197

197-
if (item == "nereids") {
198+
/*if (item == "nereids") {
198199
group_commit_insert """ insert into ${table}(id, name, score) values(10 + 1, 'h', 100); """, 1
199200
group_commit_insert """ insert into ${table}(id, name, score) select 10 + 2, 'h', 100; """, 1
200201
group_commit_insert """ insert into ${table} with label test_gc_""" + System.currentTimeMillis() + """ (id, name, score) values(13, 'h', 100); """, 1
201202
getRowCount(23)
202-
} else {
203+
} else {*/
203204
none_group_commit_insert """ insert into ${table}(id, name, score) values(10 + 1, 'h', 100); """, 1
204205
none_group_commit_insert """ insert into ${table}(id, name, score) select 10 + 2, 'h', 100; """, 1
205206
none_group_commit_insert """ insert into ${table} with label test_gc_""" + System.currentTimeMillis() + """ (id, name, score) values(13, 'h', 100); """, 1
206-
}
207+
//}
207208

208209
def rowCount = sql "select count(*) from ${table}"
209210
logger.info("row count: " + rowCount)
@@ -297,7 +298,7 @@ suite("insert_group_commit_into") {
297298
if (item == "nereids") {
298299
sql """ set enable_nereids_dml = true; """
299300
sql """ set enable_nereids_planner=true; """
300-
sql """ set enable_fallback_to_original_planner=false; """
301+
//sql """ set enable_fallback_to_original_planner=false; """
301302
} else {
302303
sql """ set enable_nereids_dml = false; """
303304
}

0 commit comments

Comments
 (0)