- fix --restore-database-mapping behavior for `ATTACH MATERIALIZED …

…VIEW` corner cases, fix #559

Signed-off-by: Slach <bloodjazman@gmail.com>

ChangeLog.md

@@ -15,6 +15,7 @@ BUG FIXES
 - `create` and `restore` commands will respect `skip_tables` config options and `--table` cli parameter, to avoid create unnecessary empty databases, fix [583](https://github.com/AlexAkulov/clickhouse-backup/issues/583)
 - fix `watch` unexpected connection closed behavior, fix [568](https://github.com/AlexAkulov/clickhouse-backup/issues/568)
 - fix `watch` validation parameters corner cases, close [569](https://github.com/AlexAkulov/clickhouse-backup/pull/569)
+- fix `--restore-database-mapping` behavior for `ATTACH MATERIALIZED VIEW` corner cases, fix [559](https://github.com/AlexAkulov/clickhouse-backup/issues/559)
 
 # v2.1.2
 IMPROVEMENTS

pkg/backup/restore.go

@@ -523,6 +523,15 @@ func (b *Backuper) restoreDataEmbedded(backupName string, tablesForRestore ListO
 }
 
 func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, tablePattern string, tablesForRestore ListOfTables, diskMap map[string]string, disks []clickhouse.Disk, log *apexLog.Entry) error {
+	if len(b.cfg.General.RestoreDatabaseMapping) > 0 {
+		for _, targetDb := range b.cfg.General.RestoreDatabaseMapping {
+			if tablePattern != "" {
+				tablePattern += "," + targetDb + ".*"
+			} else {
+				tablePattern += targetDb + ".*"
+			}
+		}
+	}
 	chTables, err := b.ch.GetTables(ctx, tablePattern)
 	if err != nil {
 		return err
@@ -550,21 +559,16 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta
 		}
 	}
 	dstTablesMap := map[metadata.TableTitle]clickhouse.Table{}
-	for i := range chTables {
+	for i, chTable := range chTables {
 		dstTablesMap[metadata.TableTitle{
 			Database: chTables[i].Database,
 			Table:    chTables[i].Name,
-		}] = chTables[i]
+		}] = chTable
 	}
 
 	var missingTables []string
 	for _, tableForRestore := range tablesForRestore {
 		found := false
-		if len(b.cfg.General.RestoreDatabaseMapping) > 0 {
-			if targetDB, isMapped := b.cfg.General.RestoreDatabaseMapping[tableForRestore.Database]; isMapped {
-				tableForRestore.Database = targetDB
-			}
-		}
 		for _, chTable := range chTables {
 			if (tableForRestore.Database == chTable.Database) && (tableForRestore.Table == chTable.Name) {
 				found = true

pkg/backup/table_pattern.go

@@ -146,12 +146,12 @@ func getTableListByPatternLocal(cfg *config.Config, metadataPath string, tablePa
 	return result, nil
 }
 
-var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.([^\s\x60.]*)(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.))?`)
+var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.([^\s\x60.]*)(?:( UUID '[^']+'))?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.))?`)
 var createRE = regexp.MustCompile(`(?m)^CREATE`)
 var attachRE = regexp.MustCompile(`(?m)^ATTACH`)
 var uuidRE = regexp.MustCompile(`UUID '[a-f\d\-]+'`)
 
-var replicatedRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'([^\)]+)\)`)
+var replicatedRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'([^)]+)\)`)
 var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^,]+),([^)]+)\)`)
 
 func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRule map[string]string) error {
@@ -161,12 +161,24 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu
 			// substitute database in the table create query
 			var substitution string
 
-			if len(createRE.FindAllString(originTable.Query, -1)) > 0 {
+			if createRE.MatchString(originTable.Query) {
 				// matching CREATE... command
 				substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}", targetDB)
-			} else if len(attachRE.FindAllString(originTable.Query, -1)) > 0 {
-				// matching ATTACH...TO... command
-				substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}%v${11}", targetDB, targetDB)
+			} else if attachRE.MatchString(originTable.Query) {
+				matches := queryRE.FindAllStringSubmatch(originTable.Query, -1)
+				if matches[0][4] != originTable.Database {
+					return fmt.Errorf("invalid SQL: %s for restore-database-mapping[%s]=%s", originTable.Query, originTable.Database, targetDB)
+				}
+				setMatchedDb := func(clauseTargetDb string) string {
+					if clauseMappedDb, isClauseMapped := dbMapRule[clauseTargetDb]; isClauseMapped {
+						clauseTargetDb = clauseMappedDb
+					}
+					return clauseTargetDb
+				}
+				toClauseTargetDb := setMatchedDb(matches[0][10])
+				fromClauseTargetDb := setMatchedDb(matches[0][15])
+				// matching ATTACH ... TO .. SELECT ... FROM ... command
+				substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}%v${16}${17}", targetDB, toClauseTargetDb, fromClauseTargetDb)
 			} else {
 				if originTable.Query == "" {
 					continue

test/integration/install_delve.sh

@@ -24,7 +24,7 @@ CGO_ENABLED=0 GO111MODULE=on go install -ldflags "-s -w -extldflags '-static'" g
 # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- server
 # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- restore --schema test_backup_8007633179464680930
 # CLICKHOUSE_TIMEOUT=3m CLICKHOUSE_DEBUG=true LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- watch --watch-interval=1m --full-interval=2m
-
+# LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- restore --data --restore-database-mapping database1:database2 --tables database1.* test_restore_database_mapping
 # run integration_test.go under debug, run from host OS not inside docker
 # go test -timeout 30m -failfast -tags=integration -run "TestIntegrationEmbedded" -v ./test/integration/integration_test.go -c -o ./test/integration/integration_test
 # sudo -H bash -c 'export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server; export COMPOSE_FILE=docker-compose_advanced.yml; export CLICKHOUSE_VERSION=head; cd ./test/integration/; /root/go/bin/dlv --listen=127.0.0.1:40002 --headless=true --api-version=2 --accept-multiclient exec ./integration_test -- -test.timeout 30m -test.failfast -test.run "TestIntegrationEmbedded"'

test/integration/integration_test.go

@@ -693,34 +693,34 @@ func TestRestoreDatabaseMapping(t *testing.T) {
 	ch.queryWithNoError(r, "CREATE DATABASE database1")
 	ch.queryWithNoError(r, "CREATE TABLE database1.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t1','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
 	ch.queryWithNoError(r, "CREATE TABLE database1.d1 AS database1.t1 ENGINE=Distributed('{cluster}',database1, t1)")
+	ch.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
+	ch.queryWithNoError(r, "CREATE MATERIALIZED VIEW database1.mv1 TO database1.t2 AS SELECT * FROM database1.t1")
 	ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)")
 
 	log.Info("Create backup")
 	r.NoError(dockerExec("clickhouse", "clickhouse-backup", "create", testBackupName))
-	r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--restore-database-mapping", "database1:database2", testBackupName))
+	r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--schema", "--rm", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName))
+	r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--data", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName))
 
 	ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)")
 
 	log.Info("Check result")
-	result := make([]int, 0)
-	r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database2.t1"))
-	r.Equal(1, len(result), "expect one row")
-	r.Equal(10, result[0], "expect count=10")
-
-	result = make([]int, 0)
-	r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database2.d1"))
-	r.Equal(1, len(result), "expect one row")
-	r.Equal(10, result[0], "expect count=10")
-
-	result = make([]int, 0)
-	r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database1.t1"))
-	r.Equal(1, len(result), "expect one row")
-	r.Equal(20, result[0], "expect count=20")
-
-	result = make([]int, 0)
-	r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database1.d1"))
-	r.Equal(1, len(result), "expect one row")
-	r.Equal(20, result[0], "expect count=20")
+	checkRecordset := func(expectedRows, expectedCount int, query string) {
+		result := make([]int, 0)
+		r.NoError(ch.chbackend.Select(&result, query))
+		r.Equal(expectedRows, len(result), "expect %d row", expectedRows)
+		r.Equal(expectedCount, result[0], "expect count=%d", expectedCount)
+	}
+
+	// database 2
+	checkRecordset(1, 10, "SELECT count() FROM database2.t1")
+	checkRecordset(1, 10, "SELECT count() FROM database2.d1")
+	checkRecordset(1, 10, "SELECT count() FROM database2.mv1")
+
+	// database 1
+	checkRecordset(1, 20, "SELECT count() FROM database1.t1")
+	checkRecordset(1, 20, "SELECT count() FROM database1.d1")
+	checkRecordset(1, 20, "SELECT count() FROM database1.mv1")
 
 	fullCleanup(r, ch, []string{testBackupName}, []string{"local"}, databaseList, true)
 }