Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[chore](backup) Fix the db name of the restored view #37412

Merged
merged 3 commits into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ private void prepareAndSendSnapshotTask() {
}
}

backupMeta = new BackupMeta(copiedTables, copiedResources);
backupMeta = new BackupMeta(db.getName(), copiedTables, copiedResources);

// send tasks
for (AgentTask task : batchTask.getAllTasks()) {
Expand Down Expand Up @@ -992,8 +992,6 @@ public void readFields(DataInput in) throws IOException {

// table refs
int size = in.readInt();
LOG.info("read {} tablerefs ", size);

tableRefs = Lists.newArrayList();
for (int i = 0; i < size; i++) {
TableRef tblRef = TableRef.read(in);
Expand All @@ -1008,16 +1006,13 @@ public void readFields(DataInput in) throws IOException {

// snapshot info
size = in.readInt();
LOG.info("read {} snapshotinfo ", size);

for (int i = 0; i < size; i++) {
SnapshotInfo snapshotInfo = SnapshotInfo.read(in);
snapshotInfos.put(snapshotInfo.getTabletId(), snapshotInfo);
}

// backup meta
if (in.readBoolean()) {
LOG.info("read backup meta");
backupMeta = BackupMeta.read(in);
}

Expand All @@ -1033,8 +1028,6 @@ public void readFields(DataInput in) throws IOException {
}
// read properties
size = in.readInt();
LOG.info("read {} property ", size);

for (int i = 0; i < size; i++) {
String key = Text.readString(in);
String value = Text.readString(in);
Expand Down
11 changes: 9 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@
import java.util.Map;

public class BackupMeta implements Writable, GsonPostProcessable {

@SerializedName(value = "db")
private String dbName;
// tbl name -> tbl
@SerializedName(value = "tblNameMap")
private Map<String, Table> tblNameMap = Maps.newHashMap();
Expand All @@ -55,7 +56,9 @@ public class BackupMeta implements Writable, GsonPostProcessable {
private BackupMeta() {
}

public BackupMeta(List<Table> tables, List<Resource> resources) {
public BackupMeta(String dbName, List<Table> tables, List<Resource> resources) {
this.dbName = dbName;

for (Table table : tables) {
tblNameMap.put(table.getName(), table);
tblIdMap.put(table.getId(), table);
Expand All @@ -65,6 +68,10 @@ public BackupMeta(List<Table> tables, List<Resource> resources) {
}
}

public String getDbName() {
return dbName;
}

public Map<String, Table> getTables() {
return tblNameMap;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,8 @@ private void checkAndPrepareMeta() {
return;
}
} else {
remoteView.resetIdsForRestore(env);
String srcDbName = backupMeta.getDbName();
remoteView.resetIdsForRestore(env, srcDbName, db.getFullName());
restoredTbls.add(remoteView);
}
}
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,13 @@ public static View read(DataInput in) throws IOException {
return GsonUtils.GSON.fromJson(Text.readString(in), View.class);
}

public void resetIdsForRestore(Env env) {
public void resetIdsForRestore(Env env, String srcDbName, String dbName) {
id = env.getNextId();

// the source db name is not setted in old BackupMeta, keep compatible with the old one.
if (srcDbName != null) {
inlineViewDef = inlineViewDef.replaceAll(srcDbName, dbName);
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List<Bac
List<Table> tbls = Lists.newArrayList();
tbls.add(tbl);
List<Resource> resources = Lists.newArrayList();
BackupMeta backupMeta = new BackupMeta(tbls, resources);
BackupMeta backupMeta = new BackupMeta(null, tbls, resources);
Map<Long, SnapshotInfo> snapshotInfos = Maps.newHashMap();
for (Partition part : tbl.getPartitions()) {
for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ boolean await(long timeout, TimeUnit unit) {
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
tbls.add(expectedRestoreTbl);
backupMeta = new BackupMeta(tbls, resources);
backupMeta = new BackupMeta(null, tbls, resources);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
suite("test_backup_restore_with_view", "backup_restore") {
String suiteName = "backup_restore_with_view"
String dbName = "${suiteName}_db"
String dbName1 = "${suiteName}_db_1"
String repoName = "${suiteName}_repo"
String snapshotName = "${suiteName}_snapshot"
String tableName = "${suiteName}_table"
Expand All @@ -26,6 +27,7 @@ suite("test_backup_restore_with_view", "backup_restore") {
def syncer = getSyncer()
syncer.createS3Repository(repoName)
sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"

int numRows = 10;
sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE"
Expand Down Expand Up @@ -66,11 +68,11 @@ suite("test_backup_restore_with_view", "backup_restore") {
def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
assertTrue(snapshot != null)

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP VIEW ${dbName}.${viewName}"
sql "DROP TABLE IF EXISTS ${dbName1}.${tableName} FORCE"
sql "DROP VIEW IF EXISTS ${dbName1}.${viewName}"

sql """
RESTORE SNAPSHOT ${dbName}.${snapshotName}
RESTORE SNAPSHOT ${dbName1}.${snapshotName}
FROM `${repoName}`
PROPERTIES
(
Expand All @@ -79,14 +81,24 @@ suite("test_backup_restore_with_view", "backup_restore") {
)
"""

syncer.waitAllRestoreFinish(dbName)
syncer.waitAllRestoreFinish(dbName1)

qt_sql "SELECT * FROM ${dbName1}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName1}.${viewName} ORDER BY id ASC"
def show_view_result = sql_return_maparray "SHOW VIEW FROM ${tableName} FROM ${dbName1}"
logger.info("show view result: ${show_view_result}")
assertTrue(show_view_result.size() == 1);
def show_view = show_view_result[0]['Create View']
assertTrue(show_view.contains("${dbName1}"))
assertTrue(show_view.contains("${tableName}"))

qt_sql "SELECT * FROM ${dbName}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName}.${viewName} ORDER BY id ASC"

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP VIEW ${dbName}.${viewName}"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP TABLE ${dbName1}.${tableName} FORCE"
sql "DROP VIEW ${dbName1}.${viewName}"
sql "DROP DATABASE ${dbName1} FORCE"
sql "DROP REPOSITORY `${repoName}`"
}

Loading