Skip to content

Commit d29cb5b

Browse files
SWJTU-ZhangLeiLchangliang
SWJTU-ZhangLei
authored and
Lchangliang
committed
[bugfix](schema change) fix multi alter clauses for light schema change. (#2)
1 parent f231fcc commit d29cb5b

File tree

7 files changed

+52
-28
lines changed

7 files changed

+52
-28
lines changed

‎be/src/olap/tablet_schema.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id,
527527
bool has_bf_columns = false;
528528
_cols.clear();
529529
_field_name_to_index.clear();
530+
_field_id_to_index.clear();
530531

531532
for (const POlapTableIndexSchema& index : ptable_schema_param.indexes()) {
532533
if (index.id() == index_id) {

‎fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,7 @@ protected void runWaitingTxnJob() throws AlterCancelException {
385385
AlterReplicaTask rollupTask = new AlterReplicaTask(rollupReplica.getBackendId(), dbId, tableId,
386386
partitionId, rollupIndexId, baseIndexId, rollupTabletId, baseTabletId,
387387
rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, visibleVersion, jobId,
388-
JobType.ROLLUP, defineExprs, descTable);
388+
JobType.ROLLUP, defineExprs, descTable, null);
389389
rollupBatchTask.addTask(rollupTask);
390390
}
391391
}

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

+28-22
Original file line numberDiff line numberDiff line change
@@ -1578,6 +1578,8 @@ public void process(List<AlterClause> alterClauses, String clusterName, Database
15781578
throws UserException {
15791579
olapTable.writeLockOrDdlException();
15801580
try {
1581+
//alterClauses can or cannot light schema change
1582+
boolean ligthSchemaChange = true;
15811583
// index id -> index schema
15821584
Map<Long, List<Column>> indexSchemaMap = new HashMap<>();
15831585
for (Map.Entry<Long, List<Column>> entry : olapTable.getIndexIdToSchema(true).entrySet()) {
@@ -1654,55 +1656,59 @@ public void process(List<AlterClause> alterClauses, String clusterName, Database
16541656

16551657
if (alterClause instanceof AddColumnClause) {
16561658
// add column
1657-
boolean ligthSchemaChange = processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap);
1658-
LOG.debug("processAddColumn, table: {}({}), getMaxColUniqueId(): {}, ligthSchemaChange: {}", olapTable.getName(), olapTable.getId(),
1659-
olapTable.getMaxColUniqueId(), ligthSchemaChange);
1660-
if (ligthSchemaChange) {
1661-
//for schema change add column optimize, direct modify table meta.
1662-
Catalog.getCurrentCatalog().modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, false);
1663-
return;
1659+
boolean clauseCanLigthSchemaChange = processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap);
1660+
if (clauseCanLigthSchemaChange == false) {
1661+
ligthSchemaChange = false;
16641662
}
16651663
} else if (alterClause instanceof AddColumnsClause) {
16661664
// add columns
1667-
boolean ligthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap, false);
1668-
LOG.debug("processAddColumns, table: {}({}), getMaxColUniqueId(): {}, ligthSchemaChange: {}", olapTable.getName(), olapTable.getId(),
1669-
olapTable.getMaxColUniqueId(), ligthSchemaChange);
1670-
if (ligthSchemaChange) {
1671-
//for schema change add column optimize, direct modify table meta.
1672-
Catalog.getCurrentCatalog().modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, false);
1673-
return;
1665+
boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap, false);
1666+
if (clauseCanLigthSchemaChange == false) {
1667+
ligthSchemaChange = false;
16741668
}
16751669
} else if (alterClause instanceof DropColumnClause) {
16761670
// drop column and drop indexes on this column
1677-
boolean ligthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, newIndexes);
1678-
LOG.debug("processDropColumn, table: {}({}), getMaxColUniqueId(): {}", olapTable.getName(), olapTable.getId(), olapTable.getMaxColUniqueId());
1679-
if (ligthSchemaChange) {
1680-
//for schema change add column optimize, direct modify table meta.
1681-
Catalog.getCurrentCatalog().modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, false);
1682-
return;
1671+
boolean clauseCanLigthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, newIndexes);
1672+
if (clauseCanLigthSchemaChange == false) {
1673+
ligthSchemaChange = false;
16831674
}
16841675
} else if (alterClause instanceof ModifyColumnClause) {
16851676
// modify column
16861677
processModifyColumn((ModifyColumnClause) alterClause, olapTable, indexSchemaMap);
1678+
ligthSchemaChange = false;
16871679
} else if (alterClause instanceof ReorderColumnsClause) {
16881680
// reorder column
16891681
processReorderColumn((ReorderColumnsClause) alterClause, olapTable, indexSchemaMap);
1682+
ligthSchemaChange = false;
16901683
} else if (alterClause instanceof ModifyTablePropertiesClause) {
16911684
// modify table properties
16921685
// do nothing, properties are already in propertyMap
1686+
ligthSchemaChange = false;
16931687
} else if (alterClause instanceof CreateIndexClause) {
16941688
if (processAddIndex((CreateIndexClause) alterClause, olapTable, newIndexes)) {
16951689
return;
16961690
}
1691+
ligthSchemaChange = false;
16971692
} else if (alterClause instanceof DropIndexClause) {
16981693
if (processDropIndex((DropIndexClause) alterClause, olapTable, newIndexes)) {
16991694
return;
17001695
}
1696+
ligthSchemaChange = false;
17011697
} else {
17021698
Preconditions.checkState(false);
17031699
}
1704-
} // end for alter clausesnnnnnn
1705-
createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes);
1700+
} // end for alter clauses
1701+
1702+
LOG.debug("processAddColumns, table: {}({}), getMaxColUniqueId(): {}, ligthSchemaChange: {}", olapTable.getName(),
1703+
olapTable.getId(), olapTable.getMaxColUniqueId(), ligthSchemaChange);
1704+
1705+
if (ligthSchemaChange) {
1706+
//for schema change add/drop value column optimize, direct modify table meta.
1707+
Catalog.getCurrentCatalog().modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, false);
1708+
return;
1709+
} else {
1710+
createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes);
1711+
}
17061712
} finally {
17071713
olapTable.writeUnlock();
17081714
}

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -437,7 +437,7 @@ protected void runWaitingTxnJob() throws AlterCancelException {
437437
long originIdxId = indexIdMap.get(shadowIdxId);
438438
int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
439439
int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId));
440-
440+
List<Column> originSchemaColumns = tbl.getSchemaByIndexId(originIdxId);
441441
for (Tablet shadowTablet : shadowIdx.getTablets()) {
442442
long shadowTabletId = shadowTablet.getId();
443443
long originTabletId = partitionIndexTabletMap.get(partitionId, shadowIdxId).get(shadowTabletId);
@@ -446,7 +446,7 @@ protected void runWaitingTxnJob() throws AlterCancelException {
446446
AlterReplicaTask rollupTask = new AlterReplicaTask(shadowReplica.getBackendId(), dbId,
447447
tableId, partitionId, shadowIdxId, originIdxId, shadowTabletId, originTabletId,
448448
shadowReplica.getId(), shadowSchemaHash, originSchemaHash, visibleVersion, jobId,
449-
JobType.SCHEMA_CHANGE, defineExprs, descTable);
449+
JobType.SCHEMA_CHANGE, defineExprs, descTable, originSchemaColumns);
450450
schemaChangeBatchTask.addTask(rollupTask);
451451
}
452452
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public SlotRef(TableName tblName, String col) {
7171
public SlotRef(SlotDescriptor desc) {
7272
super();
7373
this.tblName = null;
74-
this.col = null;
74+
this.col = desc.getColumn() != null ? desc.getColumn().getName() : null;
7575
this.desc = desc;
7676
this.type = desc.getType();
7777
// TODO(zc): label is meaningful

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,11 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) {
190190
List<String> columns = Lists.newArrayList();
191191
List<TColumn> columns_desc = Lists.newArrayList();
192192
columns.addAll(indexMeta.getSchema().stream().map(Column::getName).collect(Collectors.toList()));
193-
columns_desc.addAll(indexMeta.getSchema().stream().map(Column::toThrift).collect(Collectors.toList()));
193+
for (Column column : indexMeta.getSchema()) {
194+
TColumn tColumn = column.toThrift();
195+
column.setIndexFlag(tColumn, table.getIndexes());
196+
columns_desc.add(tColumn);
197+
}
194198
TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns,
195199
indexMeta.getSchemaHash(), columns_desc);
196200
schemaParam.addToIndexes(indexSchema);

‎fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java

+14-1
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@
2121
import org.apache.doris.analysis.DescriptorTable;
2222
import org.apache.doris.analysis.Expr;
2323
import org.apache.doris.analysis.SlotRef;
24+
import org.apache.doris.catalog.Column;
2425
import org.apache.doris.thrift.TAlterMaterializedViewParam;
2526
import org.apache.doris.thrift.TAlterTabletReqV2;
27+
import org.apache.doris.thrift.TColumn;
2628
import org.apache.doris.thrift.TTaskType;
2729

2830
import com.google.common.collect.Lists;
2931

32+
import java.util.ArrayList;
3033
import java.util.List;
3134
import java.util.Map;
3235

@@ -48,6 +51,7 @@ public class AlterReplicaTask extends AgentTask {
4851

4952
private Map<String, Expr> defineExprs;
5053
private DescriptorTable descTable;
54+
private List<Column> baseSchemaColumns;
5155

5256
/**
5357
* AlterReplicaTask constructor.
@@ -56,7 +60,7 @@ public class AlterReplicaTask extends AgentTask {
5660
public AlterReplicaTask(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId,
5761
long baseIndexId, long rollupTabletId, long baseTabletId, long newReplicaId, int newSchemaHash,
5862
int baseSchemaHash, long version, long jobId, AlterJobV2.JobType jobType, Map<String, Expr> defineExprs,
59-
DescriptorTable descTable) {
63+
DescriptorTable descTable, List<Column> baseSchemaColumns) {
6064
super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId);
6165

6266
this.baseTabletId = baseTabletId;
@@ -71,6 +75,7 @@ public AlterReplicaTask(long backendId, long dbId, long tableId, long partitionI
7175
this.jobType = jobType;
7276
this.defineExprs = defineExprs;
7377
this.descTable = descTable;
78+
this.baseSchemaColumns = baseSchemaColumns;
7479
}
7580

7681
public long getBaseTabletId() {
@@ -115,6 +120,14 @@ public TAlterTabletReqV2 toThrift() {
115120
}
116121
}
117122
req.setDescTbl(descTable.toThrift());
123+
124+
if (baseSchemaColumns != null) {
125+
List<TColumn> columns = new ArrayList<TColumn>();
126+
for (Column column : baseSchemaColumns) {
127+
columns.add(column.toThrift());
128+
}
129+
req.setColumns(columns);
130+
}
118131
return req;
119132
}
120133
}

0 commit comments

Comments
 (0)