Skip to content

Commit 47c163e

Browse files
LchangliangLchangliang
committed
fix schema change test
1 parent db42352 commit 47c163e

16 files changed

+650
-47
lines changed

‎be/src/olap/compaction.cpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,9 @@ Status Compaction::do_compaction_impl(int64_t permits) {
145145
LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name()
146146
<< ", output_version=" << _output_version << ", permits: " << permits;
147147
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
148-
const TabletSchema* cur_tablet_schema = &_tablet->tablet_schema();
148+
const TabletSchema cur_tablet_schema = _tablet->tablet_schema();
149149

150-
RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema));
150+
RETURN_NOT_OK(construct_output_rowset_writer(&cur_tablet_schema));
151151
RETURN_NOT_OK(construct_input_rowset_readers());
152152
TRACE("prepare finished");
153153

@@ -157,10 +157,10 @@ Status Compaction::do_compaction_impl(int64_t permits) {
157157
Status res;
158158

159159
if (use_vectorized_compaction) {
160-
res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema,
160+
res = Merger::vmerge_rowsets(_tablet, compaction_type(), &cur_tablet_schema,
161161
_input_rs_readers, _output_rs_writer.get(), &stats);
162162
} else {
163-
res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema,
163+
res = Merger::merge_rowsets(_tablet, compaction_type(), &cur_tablet_schema,
164164
_input_rs_readers, _output_rs_writer.get(), &stats);
165165
}
166166

‎be/src/olap/snapshot_manager.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,8 @@ Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb,
232232
context.tablet_schema_hash = org_rowset_meta->tablet_schema_hash();
233233
context.rowset_type = org_rowset_meta->rowset_type();
234234
context.tablet_path = new_tablet_path;
235-
context.tablet_schema = &tablet_schema;
235+
context.tablet_schema =
236+
org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : &tablet_schema;
236237
context.rowset_state = org_rowset_meta->rowset_state();
237238
context.version = org_rowset_meta->version();
238239
context.oldest_write_timestamp = org_rowset_meta->oldest_write_timestamp();

‎regression-test/data/schema_change/test_partition_schema_change.out

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !test_partition_schema_change --
3+
0
4+
5+
-- !test_partition_schema_change_2 --
6+
0
7+
8+
-- !test_partition_schema_change_3 --
9+
1
10+
11+
-- !test_partition_schema_change_4 --
12+
1
13+
14+
-- !test_partition_schema_change_5 --
15+
1
16+
17+
-- !test_partition_schema_change_6 --
18+
1 2017-01-02 Beijing 10 1 2017-01-02T00:00 1 30 20
19+
1 2017-02-02 Beijing 10 1 2017-02-02T00:00 1 30 20
20+
1 2017-03-02 Beijing 10 1 2017-03-02T00:00 1 30 20
21+
22+
-- !test_partition_schema_change_7 --
23+
0
24+
25+
-- !test_partition_schema_change_8 --
26+
1
27+
28+
-- !test_partition_schema_change_9 --
29+
1 2017-01-02 Beijing 10 1 2017-01-02T00:00 1 30 20 1
30+
1 2017-02-02 Beijing 10 1 2017-02-02T00:00 1 30 20 1
31+
2 2017-02-03 Beijing 10 1 2017-02-02T00:00 1 30 20 2
32+
1 2017-03-02 Beijing 10 1 2017-03-02T00:00 1 30 20 1
33+

‎regression-test/suites/schema_change/test_agg_keys_schema_change.groovy

+48-3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,42 @@ suite ("test_agg_keys_schema_change") {
2121
def tableName = "schema_change_agg_keys_regression_test"
2222

2323
try {
24+
25+
String[][] backends = sql """ show backends; """
26+
assertTrue(backends.size() > 0)
27+
String backend_id;
28+
def backendId_to_backendIP = [:]
29+
def backendId_to_backendHttpPort = [:]
30+
for (String[] backend in backends) {
31+
backendId_to_backendIP.put(backend[0], backend[2])
32+
backendId_to_backendHttpPort.put(backend[0], backend[5])
33+
}
34+
35+
backend_id = backendId_to_backendIP.keySet()[0]
36+
StringBuilder showConfigCommand = new StringBuilder();
37+
showConfigCommand.append("curl -X GET http://")
38+
showConfigCommand.append(backendId_to_backendIP.get(backend_id))
39+
showConfigCommand.append(":")
40+
showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id))
41+
showConfigCommand.append("/api/show_config")
42+
logger.info(showConfigCommand.toString())
43+
def process = showConfigCommand.toString().execute()
44+
int code = process.waitFor()
45+
String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
46+
String out = process.getText()
47+
logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
48+
assertEquals(code, 0)
49+
def configList = parseJson(out.trim())
50+
assert configList instanceof List
51+
52+
boolean disableAutoCompaction = true
53+
for (Object ele in (List) configList) {
54+
assert ele instanceof List<String>
55+
if (((List<String>) ele)[0] == "disable_auto_compaction") {
56+
disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
57+
}
58+
}
59+
2460
sql """ DROP TABLE IF EXISTS ${tableName} """
2561
sql """
2662
CREATE TABLE ${tableName} (
@@ -151,10 +187,13 @@ suite ("test_agg_keys_schema_change") {
151187
String[][] tablets = sql """ show tablets from ${tableName}; """
152188
for (String[] tablet in tablets) {
153189
String tablet_id = tablet[0]
190+
backend_id = tablet[2]
154191
logger.info("run compaction:" + tablet_id)
155192
StringBuilder sb = new StringBuilder();
156193
sb.append("curl -X POST http://")
157-
sb.append(context.config.beHttpAddress)
194+
sb.append(backendId_to_backendIP.get(backend_id))
195+
sb.append(":")
196+
sb.append(backendId_to_backendHttpPort.get(backend_id))
158197
sb.append("/api/compaction/run?tablet_id=")
159198
sb.append(tablet_id)
160199
sb.append("&compact_type=cumulative")
@@ -173,9 +212,12 @@ suite ("test_agg_keys_schema_change") {
173212
do {
174213
Thread.sleep(1000)
175214
String tablet_id = tablet[0]
215+
backend_id = tablet[2]
176216
StringBuilder sb = new StringBuilder();
177217
sb.append("curl -X GET http://")
178-
sb.append(context.config.beHttpAddress)
218+
sb.append(backendId_to_backendIP.get(backend_id))
219+
sb.append(":")
220+
sb.append(backendId_to_backendHttpPort.get(backend_id))
179221
sb.append("/api/compaction/run_status?tablet_id=")
180222
sb.append(tablet_id)
181223

@@ -203,9 +245,12 @@ suite ("test_agg_keys_schema_change") {
203245
int rowCount = 0
204246
for (String[] tablet in tablets) {
205247
String tablet_id = tablet[0]
248+
backend_id = tablet[2]
206249
StringBuilder sb = new StringBuilder();
207250
sb.append("curl -X GET http://")
208-
sb.append(context.config.beHttpAddress)
251+
sb.append(backendId_to_backendIP.get(backend_id))
252+
sb.append(":")
253+
sb.append(backendId_to_backendHttpPort.get(backend_id))
209254
sb.append("/api/compaction/show?tablet_id=")
210255
sb.append(tablet_id)
211256
String command = sb.toString()

‎regression-test/suites/schema_change/test_agg_mv_schema_change.groovy

+47-3
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,41 @@ suite ("test_agg_mv_schema_change") {
2222
def tableName = "schema_change_agg_mv_regression_test"
2323

2424
try {
25+
String[][] backends = sql """ show backends; """
26+
assertTrue(backends.size() > 0)
27+
String backend_id;
28+
def backendId_to_backendIP = [:]
29+
def backendId_to_backendHttpPort = [:]
30+
for (String[] backend in backends) {
31+
backendId_to_backendIP.put(backend[0], backend[2])
32+
backendId_to_backendHttpPort.put(backend[0], backend[5])
33+
}
34+
35+
backend_id = backendId_to_backendIP.keySet()[0]
36+
StringBuilder showConfigCommand = new StringBuilder();
37+
showConfigCommand.append("curl -X GET http://")
38+
showConfigCommand.append(backendId_to_backendIP.get(backend_id))
39+
showConfigCommand.append(":")
40+
showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id))
41+
showConfigCommand.append("/api/show_config")
42+
logger.info(showConfigCommand.toString())
43+
def process = showConfigCommand.toString().execute()
44+
int code = process.waitFor()
45+
String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
46+
String out = process.getText()
47+
logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
48+
assertEquals(code, 0)
49+
def configList = parseJson(out.trim())
50+
assert configList instanceof List
51+
52+
boolean disableAutoCompaction = true
53+
for (Object ele in (List) configList) {
54+
assert ele instanceof List<String>
55+
if (((List<String>) ele)[0] == "disable_auto_compaction") {
56+
disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
57+
}
58+
}
59+
2560
sql """ DROP TABLE IF EXISTS ${tableName} """
2661
sql """
2762
CREATE TABLE ${tableName} (
@@ -131,10 +166,13 @@ suite ("test_agg_mv_schema_change") {
131166
String[][] tablets = sql """ show tablets from ${tableName}; """
132167
for (String[] tablet in tablets) {
133168
String tablet_id = tablet[0]
169+
backend_id = tablet[2]
134170
logger.info("run compaction:" + tablet_id)
135171
StringBuilder sb = new StringBuilder();
136172
sb.append("curl -X POST http://")
137-
sb.append(context.config.beHttpAddress)
173+
sb.append(backendId_to_backendIP.get(backend_id))
174+
sb.append(":")
175+
sb.append(backendId_to_backendHttpPort.get(backend_id))
138176
sb.append("/api/compaction/run?tablet_id=")
139177
sb.append(tablet_id)
140178
sb.append("&compact_type=cumulative")
@@ -154,9 +192,12 @@ suite ("test_agg_mv_schema_change") {
154192
do {
155193
Thread.sleep(1000)
156194
String tablet_id = tablet[0]
195+
backend_id = tablet[2]
157196
StringBuilder sb = new StringBuilder();
158197
sb.append("curl -X GET http://")
159-
sb.append(context.config.beHttpAddress)
198+
sb.append(backendId_to_backendIP.get(backend_id))
199+
sb.append(":")
200+
sb.append(backendId_to_backendHttpPort.get(backend_id))
160201
sb.append("/api/compaction/run_status?tablet_id=")
161202
sb.append(tablet_id)
162203

@@ -184,9 +225,12 @@ suite ("test_agg_mv_schema_change") {
184225
int rowCount = 0
185226
for (String[] tablet in tablets) {
186227
String tablet_id = tablet[0]
228+
backend_id = tablet[2]
187229
StringBuilder sb = new StringBuilder();
188230
sb.append("curl -X GET http://")
189-
sb.append(context.config.beHttpAddress)
231+
sb.append(backendId_to_backendIP.get(backend_id))
232+
sb.append(":")
233+
sb.append(backendId_to_backendHttpPort.get(backend_id))
190234
sb.append("/api/compaction/show?tablet_id=")
191235
sb.append(tablet_id)
192236
String command = sb.toString()

‎regression-test/suites/schema_change/test_agg_rollup_schema_change.groovy

+48-3
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,42 @@ suite ("test_agg_rollup_schema_change") {
2222
def tableName = "schema_change_agg_rollup_regression_test"
2323

2424
try {
25+
26+
String[][] backends = sql """ show backends; """
27+
assertTrue(backends.size() > 0)
28+
String backend_id;
29+
def backendId_to_backendIP = [:]
30+
def backendId_to_backendHttpPort = [:]
31+
for (String[] backend in backends) {
32+
backendId_to_backendIP.put(backend[0], backend[2])
33+
backendId_to_backendHttpPort.put(backend[0], backend[5])
34+
}
35+
36+
backend_id = backendId_to_backendIP.keySet()[0]
37+
StringBuilder showConfigCommand = new StringBuilder();
38+
showConfigCommand.append("curl -X GET http://")
39+
showConfigCommand.append(backendId_to_backendIP.get(backend_id))
40+
showConfigCommand.append(":")
41+
showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id))
42+
showConfigCommand.append("/api/show_config")
43+
logger.info(showConfigCommand.toString())
44+
def process = showConfigCommand.toString().execute()
45+
int code = process.waitFor()
46+
String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
47+
String out = process.getText()
48+
logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
49+
assertEquals(code, 0)
50+
def configList = parseJson(out.trim())
51+
assert configList instanceof List
52+
53+
boolean disableAutoCompaction = true
54+
for (Object ele in (List) configList) {
55+
assert ele instanceof List<String>
56+
if (((List<String>) ele)[0] == "disable_auto_compaction") {
57+
disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
58+
}
59+
}
60+
2561
sql """ DROP TABLE IF EXISTS ${tableName} """
2662
sql """
2763
CREATE TABLE ${tableName} (
@@ -131,10 +167,13 @@ suite ("test_agg_rollup_schema_change") {
131167
String[][] tablets = sql """ show tablets from ${tableName}; """
132168
for (String[] tablet in tablets) {
133169
String tablet_id = tablet[0]
170+
backend_id = tablet[2]
134171
logger.info("run compaction:" + tablet_id)
135172
StringBuilder sb = new StringBuilder();
136173
sb.append("curl -X POST http://")
137-
sb.append(context.config.beHttpAddress)
174+
sb.append(backendId_to_backendIP.get(backend_id))
175+
sb.append(":")
176+
sb.append(backendId_to_backendHttpPort.get(backend_id))
138177
sb.append("/api/compaction/run?tablet_id=")
139178
sb.append(tablet_id)
140179
sb.append("&compact_type=cumulative")
@@ -154,9 +193,12 @@ suite ("test_agg_rollup_schema_change") {
154193
do {
155194
Thread.sleep(1000)
156195
String tablet_id = tablet[0]
196+
backend_id = tablet[2]
157197
StringBuilder sb = new StringBuilder();
158198
sb.append("curl -X GET http://")
159-
sb.append(context.config.beHttpAddress)
199+
sb.append(backendId_to_backendIP.get(backend_id))
200+
sb.append(":")
201+
sb.append(backendId_to_backendHttpPort.get(backend_id))
160202
sb.append("/api/compaction/run_status?tablet_id=")
161203
sb.append(tablet_id)
162204

@@ -184,9 +226,12 @@ suite ("test_agg_rollup_schema_change") {
184226
int rowCount = 0
185227
for (String[] tablet in tablets) {
186228
String tablet_id = tablet[0]
229+
backend_id = tablet[2]
187230
StringBuilder sb = new StringBuilder();
188231
sb.append("curl -X GET http://")
189-
sb.append(context.config.beHttpAddress)
232+
sb.append(backendId_to_backendIP.get(backend_id))
233+
sb.append(":")
234+
sb.append(backendId_to_backendHttpPort.get(backend_id))
190235
sb.append("/api/compaction/show?tablet_id=")
191236
sb.append(tablet_id)
192237
String command = sb.toString()

0 commit comments

Comments
 (0)