From 8e416b9fd9f7bf19a3066e04c9b6bab0a1a97b36 Mon Sep 17 00:00:00 2001 From: w41ter Date: Fri, 5 Jul 2024 09:42:48 +0000 Subject: [PATCH 1/2] [chore](backup) Fix the db name of the restored view Previously, during restore, the database name in the CREATE VIEW statement was not modified, causing the restored view to be unviewable with the SHOW VIEW command. This PR retains the original cluster's database name in the BackupMeta and manually replaces it with the new cluster's database name in the CREATE VIEW statement during restore. --- .../main/java/org/apache/doris/backup/BackupJob.java | 9 +-------- .../main/java/org/apache/doris/backup/BackupMeta.java | 11 +++++++++-- .../main/java/org/apache/doris/backup/RestoreJob.java | 3 ++- .../src/main/java/org/apache/doris/catalog/View.java | 7 ++++++- .../org/apache/doris/backup/BackupHandlerTest.java | 2 +- .../java/org/apache/doris/backup/RestoreJobTest.java | 2 +- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 008948636503d4..c706fb7075338d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -454,7 +454,7 @@ private void prepareAndSendSnapshotTask() { } } - backupMeta = new BackupMeta(copiedTables, copiedResources); + backupMeta = new BackupMeta(db.getName(), copiedTables, copiedResources); // send tasks for (AgentTask task : batchTask.getAllTasks()) { @@ -992,8 +992,6 @@ public void readFields(DataInput in) throws IOException { // table refs int size = in.readInt(); - LOG.info("read {} tablerefs ", size); - tableRefs = Lists.newArrayList(); for (int i = 0; i < size; i++) { TableRef tblRef = TableRef.read(in); @@ -1008,8 +1006,6 @@ public void readFields(DataInput in) throws IOException { // snapshot info size = in.readInt(); - LOG.info("read {} snapshotinfo ", size); - for (int i = 0; i < size; i++) { SnapshotInfo snapshotInfo = SnapshotInfo.read(in); snapshotInfos.put(snapshotInfo.getTabletId(), snapshotInfo); @@ -1017,7 +1013,6 @@ public void readFields(DataInput in) throws IOException { // backup meta if (in.readBoolean()) { - LOG.info("read backup meta"); backupMeta = BackupMeta.read(in); } @@ -1033,8 +1028,6 @@ public void readFields(DataInput in) throws IOException { } // read properties size = in.readInt(); - LOG.info("read {} property ", size); - for (int i = 0; i < size; i++) { String key = Text.readString(in); String value = Text.readString(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index e27c8d19a840a2..c08d8c160027c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -42,7 +42,8 @@ import java.util.Map; public class BackupMeta implements Writable, GsonPostProcessable { - + @SerializedName(value = "db") + private String dbName; // tbl name -> tbl @SerializedName(value = "tblNameMap") private Map tblNameMap = Maps.newHashMap(); @@ -55,7 +56,9 @@ public class BackupMeta implements Writable, GsonPostProcessable { private BackupMeta() { } - public BackupMeta(List tables, List resources) { + public BackupMeta(String dbName, List
tables, List resources) { + this.dbName = dbName; + for (Table table : tables) { tblNameMap.put(table.getName(), table); tblIdMap.put(table.getId(), table); @@ -65,6 +68,10 @@ public BackupMeta(List
tables, List resources) { } } + public String getDbName() { + return dbName; + } + public Map getTables() { return tblNameMap; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 098c473d32e22e..ac94e03f2524d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -783,7 +783,8 @@ private void checkAndPrepareMeta() { return; } } else { - remoteView.resetIdsForRestore(env); + String srcDbName = backupMeta.getDbName(); + remoteView.resetIdsForRestore(env, srcDbName, db.getFullName()); restoredTbls.add(remoteView); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java index 4d2558f8748ab8..8285dedc0941b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java @@ -250,8 +250,13 @@ public static View read(DataInput in) throws IOException { return GsonUtils.GSON.fromJson(Text.readString(in), View.class); } - public void resetIdsForRestore(Env env) { + public void resetIdsForRestore(Env env, String srcDbName, String dbName) { id = env.getNextId(); + + // the source db name is not setted in old BackupMeta, keep compatible with the old one. + if (srcDbName != null) { + inlineViewDef = inlineViewDef.replaceAll(srcDbName, dbName); + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java index 97e689b697256c..5de2be6c9ee5f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java @@ -212,7 +212,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List tbls = Lists.newArrayList(); tbls.add(tbl); List resources = Lists.newArrayList(); - BackupMeta backupMeta = new BackupMeta(tbls, resources); + BackupMeta backupMeta = new BackupMeta(null, tbls, resources); Map snapshotInfos = Maps.newHashMap(); for (Partition part : tbl.getPartitions()) { for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index 71cad0438c9022..43bdc5c2675cf9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -258,7 +258,7 @@ boolean await(long timeout, TimeUnit unit) { List
tbls = Lists.newArrayList(); List resources = Lists.newArrayList(); tbls.add(expectedRestoreTbl); - backupMeta = new BackupMeta(tbls, resources); + backupMeta = new BackupMeta(null, tbls, resources); } @Test From e589c2c67237408853c5f22cd1a02f32c5c78fbf Mon Sep 17 00:00:00 2001 From: w41ter Date: Mon, 15 Jul 2024 11:09:49 +0000 Subject: [PATCH 2/2] [chore](cases) Add the db name of the restored view checking case --- ....out => test_backup_restore_with_view.out} | 0 ...y => test_backup_restore_with_view.groovy} | 24 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) rename regression-test/data/backup_restore/{test_backup_restore_with.out => test_backup_restore_with_view.out} (100%) rename regression-test/suites/backup_restore/{test_backup_restore_with.groovy => test_backup_restore_with_view.groovy} (74%) diff --git a/regression-test/data/backup_restore/test_backup_restore_with.out b/regression-test/data/backup_restore/test_backup_restore_with_view.out similarity index 100% rename from regression-test/data/backup_restore/test_backup_restore_with.out rename to regression-test/data/backup_restore/test_backup_restore_with_view.out diff --git a/regression-test/suites/backup_restore/test_backup_restore_with.groovy b/regression-test/suites/backup_restore/test_backup_restore_with_view.groovy similarity index 74% rename from regression-test/suites/backup_restore/test_backup_restore_with.groovy rename to regression-test/suites/backup_restore/test_backup_restore_with_view.groovy index cacb9df6de54c8..ae277a4cd931ba 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_with.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_with_view.groovy @@ -18,6 +18,7 @@ suite("test_backup_restore_with_view", "backup_restore") { String suiteName = "backup_restore_with_view" String dbName = "${suiteName}_db" + String dbName1 = "${suiteName}_db_1" String repoName = "${suiteName}_repo" String snapshotName = "${suiteName}_snapshot" String tableName = "${suiteName}_table" @@ -26,6 +27,7 @@ suite("test_backup_restore_with_view", "backup_restore") { def syncer = getSyncer() syncer.createS3Repository(repoName) sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "CREATE DATABASE IF NOT EXISTS ${dbName1}" int numRows = 10; sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE" @@ -66,11 +68,11 @@ suite("test_backup_restore_with_view", "backup_restore") { def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) assertTrue(snapshot != null) - sql "DROP TABLE ${dbName}.${tableName} FORCE" - sql "DROP VIEW ${dbName}.${viewName}" + sql "DROP TABLE IF EXISTS ${dbName1}.${tableName} FORCE" + sql "DROP VIEW IF EXISTS ${dbName1}.${viewName}" sql """ - RESTORE SNAPSHOT ${dbName}.${snapshotName} + RESTORE SNAPSHOT ${dbName1}.${snapshotName} FROM `${repoName}` PROPERTIES ( @@ -79,14 +81,24 @@ suite("test_backup_restore_with_view", "backup_restore") { ) """ - syncer.waitAllRestoreFinish(dbName) + syncer.waitAllRestoreFinish(dbName1) + + qt_sql "SELECT * FROM ${dbName1}.${tableName} ORDER BY id ASC" + qt_sql "SELECT * FROM ${dbName1}.${viewName} ORDER BY id ASC" + def show_view_result = sql_return_maparray "SHOW VIEW FROM ${tableName} FROM ${dbName1}" + logger.info("show view result: ${show_view_result}") + assertTrue(show_view_result.size() == 1); + def show_view = show_view_result[0]['Create View'] + assertTrue(show_view.contains("${dbName1}")) + assertTrue(show_view.contains("${tableName}")) - qt_sql "SELECT * FROM ${dbName}.${tableName} ORDER BY id ASC" - qt_sql "SELECT * FROM ${dbName}.${viewName} ORDER BY id ASC" sql "DROP TABLE ${dbName}.${tableName} FORCE" sql "DROP VIEW ${dbName}.${viewName}" sql "DROP DATABASE ${dbName} FORCE" + sql "DROP TABLE ${dbName1}.${tableName} FORCE" + sql "DROP VIEW ${dbName1}.${viewName}" + sql "DROP DATABASE ${dbName1} FORCE" sql "DROP REPOSITORY `${repoName}`" }