Skip to content

Commit

Permalink
[chore](backup) Fix the db name of the restored view (#37412)
Browse files Browse the repository at this point in the history
Previously, during restore, the database name in the CREATE VIEW
statement was not modified, causing the restored view to be unviewable
with the SHOW VIEW command. This PR retains the original cluster's
database name in the BackupMeta and manually replaces it with the new
cluster's database name in the CREATE VIEW statement during restore.
  • Loading branch information
w41ter authored and dataroaring committed Jul 17, 2024
1 parent 351d458 commit 7a9809d
Show file tree
Hide file tree
Showing 8 changed files with 38 additions and 20 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ private void prepareAndSendSnapshotTask() {
}
}

backupMeta = new BackupMeta(copiedTables, copiedResources);
backupMeta = new BackupMeta(db.getName(), copiedTables, copiedResources);

// send tasks
for (AgentTask task : batchTask.getAllTasks()) {
Expand Down Expand Up @@ -992,8 +992,6 @@ public void readFields(DataInput in) throws IOException {

// table refs
int size = in.readInt();
LOG.info("read {} tablerefs ", size);

tableRefs = Lists.newArrayList();
for (int i = 0; i < size; i++) {
TableRef tblRef = TableRef.read(in);
Expand All @@ -1008,16 +1006,13 @@ public void readFields(DataInput in) throws IOException {

// snapshot info
size = in.readInt();
LOG.info("read {} snapshotinfo ", size);

for (int i = 0; i < size; i++) {
SnapshotInfo snapshotInfo = SnapshotInfo.read(in);
snapshotInfos.put(snapshotInfo.getTabletId(), snapshotInfo);
}

// backup meta
if (in.readBoolean()) {
LOG.info("read backup meta");
backupMeta = BackupMeta.read(in);
}

Expand All @@ -1033,8 +1028,6 @@ public void readFields(DataInput in) throws IOException {
}
// read properties
size = in.readInt();
LOG.info("read {} property ", size);

for (int i = 0; i < size; i++) {
String key = Text.readString(in);
String value = Text.readString(in);
Expand Down
11 changes: 9 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@
import java.util.Map;

public class BackupMeta implements Writable, GsonPostProcessable {

@SerializedName(value = "db")
private String dbName;
// tbl name -> tbl
@SerializedName(value = "tblNameMap")
private Map<String, Table> tblNameMap = Maps.newHashMap();
Expand All @@ -55,7 +56,9 @@ public class BackupMeta implements Writable, GsonPostProcessable {
private BackupMeta() {
}

public BackupMeta(List<Table> tables, List<Resource> resources) {
public BackupMeta(String dbName, List<Table> tables, List<Resource> resources) {
this.dbName = dbName;

for (Table table : tables) {
tblNameMap.put(table.getName(), table);
tblIdMap.put(table.getId(), table);
Expand All @@ -65,6 +68,10 @@ public BackupMeta(List<Table> tables, List<Resource> resources) {
}
}

public String getDbName() {
return dbName;
}

public Map<String, Table> getTables() {
return tblNameMap;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,8 @@ private void checkAndPrepareMeta() {
return;
}
} else {
remoteView.resetIdsForRestore(env);
String srcDbName = backupMeta.getDbName();
remoteView.resetIdsForRestore(env, srcDbName, db.getFullName());
restoredTbls.add(remoteView);
}
}
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,13 @@ public static View read(DataInput in) throws IOException {
return GsonUtils.GSON.fromJson(Text.readString(in), View.class);
}

public void resetIdsForRestore(Env env) {
public void resetIdsForRestore(Env env, String srcDbName, String dbName) {
id = env.getNextId();

// the source db name is not setted in old BackupMeta, keep compatible with the old one.
if (srcDbName != null) {
inlineViewDef = inlineViewDef.replaceAll(srcDbName, dbName);
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List<Bac
List<Table> tbls = Lists.newArrayList();
tbls.add(tbl);
List<Resource> resources = Lists.newArrayList();
BackupMeta backupMeta = new BackupMeta(tbls, resources);
BackupMeta backupMeta = new BackupMeta(null, tbls, resources);
Map<Long, SnapshotInfo> snapshotInfos = Maps.newHashMap();
for (Partition part : tbl.getPartitions()) {
for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ boolean await(long timeout, TimeUnit unit) {
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
tbls.add(expectedRestoreTbl);
backupMeta = new BackupMeta(tbls, resources);
backupMeta = new BackupMeta(null, tbls, resources);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
suite("test_backup_restore_with_view", "backup_restore") {
String suiteName = "backup_restore_with_view"
String dbName = "${suiteName}_db"
String dbName1 = "${suiteName}_db_1"
String repoName = "${suiteName}_repo"
String snapshotName = "${suiteName}_snapshot"
String tableName = "${suiteName}_table"
Expand All @@ -26,6 +27,7 @@ suite("test_backup_restore_with_view", "backup_restore") {
def syncer = getSyncer()
syncer.createS3Repository(repoName)
sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"

int numRows = 10;
sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE"
Expand Down Expand Up @@ -66,11 +68,11 @@ suite("test_backup_restore_with_view", "backup_restore") {
def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
assertTrue(snapshot != null)

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP VIEW ${dbName}.${viewName}"
sql "DROP TABLE IF EXISTS ${dbName1}.${tableName} FORCE"
sql "DROP VIEW IF EXISTS ${dbName1}.${viewName}"

sql """
RESTORE SNAPSHOT ${dbName}.${snapshotName}
RESTORE SNAPSHOT ${dbName1}.${snapshotName}
FROM `${repoName}`
PROPERTIES
(
Expand All @@ -79,14 +81,24 @@ suite("test_backup_restore_with_view", "backup_restore") {
)
"""

syncer.waitAllRestoreFinish(dbName)
syncer.waitAllRestoreFinish(dbName1)

qt_sql "SELECT * FROM ${dbName1}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName1}.${viewName} ORDER BY id ASC"
def show_view_result = sql_return_maparray "SHOW VIEW FROM ${tableName} FROM ${dbName1}"
logger.info("show view result: ${show_view_result}")
assertTrue(show_view_result.size() == 1);
def show_view = show_view_result[0]['Create View']
assertTrue(show_view.contains("${dbName1}"))
assertTrue(show_view.contains("${tableName}"))

qt_sql "SELECT * FROM ${dbName}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName}.${viewName} ORDER BY id ASC"

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP VIEW ${dbName}.${viewName}"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP TABLE ${dbName1}.${tableName} FORCE"
sql "DROP VIEW ${dbName1}.${viewName}"
sql "DROP DATABASE ${dbName1} FORCE"
sql "DROP REPOSITORY `${repoName}`"
}

0 comments on commit 7a9809d

Please sign in to comment.