Fixed show tables size for old versions of CH

Removed 'freeze' and 'clean' commands

ReadMe.md

@@ -13,17 +13,19 @@ Tool for easy ClickHouse backup and restore with cloud storages support
 - Easy creating and restoring backups of all or specific tables
 - Efficient storing of multiple backups on the file system
 - Uploading and downloading with streaming compression
+- Works with AWS
+- **Support of Atomic Database Engine**
+- **Support of multi disks installations**
+
+TODO:
+- Works with Azure, GCS, Tencent COS, FTP
 - Support of incremental backups on remote storages
-- Works with AWS, Azure, GCS, Tencent COS, FTP
+- Smart restore for replicated tables
 
 ## Limitations
 
-- ClickHouse above 1.1.54390 and before 20.10 is supported
+- ClickHouse above 1.1.54390 is supported
 - Only MergeTree family tables engines
-- Backup of 'Tiered storage' or `storage_policy` IS NOT SUPPORTED!
-- 'Atomic' database engine enabled by default in ClickHouse 20.10 IS NOT SUPPORTED!
-- Maximum backup size on cloud storages is 5TB
-- Maximum number of parts on AWS S3 is 10,000 (increase part_size if your database is more than 1TB)
 
 ## Download
 

cmd/clickhouse-backup/main.go

@@ -114,12 +114,10 @@ func main() {
 				case "remote":
 					return backup.PrintRemoteBackups(*config, c.Args().Get(1))
 				case "all", "":
-					fmt.Println("Local backups:")
 					if err := backup.PrintLocalBackups(*config, c.Args().Get(1)); err != nil {
 						return err
 					}
 					if config.General.RemoteStorage != "none" {
-						fmt.Println("Remote backups:")
 						if err := backup.PrintRemoteBackups(*config, c.Args().Get(1)); err != nil {
 							return err
 						}
@@ -225,29 +223,6 @@ func main() {
 			},
 			Flags: cliapp.Flags,
 		},
-		{
-			Name:        "freeze",
-			Usage:       "Freeze tables",
-			UsageText:   "clickhouse-backup freeze [-t, --tables=<db>.<table>] <backup_name>",
-			Description: "Freeze tables",
-			Action: func(c *cli.Context) error {
-				return backup.Freeze(*getConfig(c), c.String("t"))
-			},
-			Flags: append(cliapp.Flags,
-				cli.StringFlag{
-					Name:   "table, tables, t",
-					Hidden: false,
-				},
-			),
-		},
-		{
-			Name:  "clean",
-			Usage: "Remove data in 'shadow' folder",
-			Action: func(c *cli.Context) error {
-				return backup.Clean(*getConfig(c))
-			},
-			Flags: cliapp.Flags,
-		},
 		{
 			Name:  "server",
 			Usage: "Run API server",

pkg/backup/backup.go

@@ -140,52 +140,6 @@ func RestoreSchema(cfg config.Config, backupName string, tablePattern string, dr
 	return nil
 }
 
-// Freeze - freeze tables by tablePattern
-func Freeze(cfg config.Config, tablePattern string) error {
-	ch := &clickhouse.ClickHouse{
-		Config: &cfg.ClickHouse,
-	}
-	if err := ch.Connect(); err != nil {
-		return fmt.Errorf("can't connect to clickhouse: %v", err)
-	}
-	defer ch.Close()
-
-	disks, err := ch.GetDisks()
-	if err != nil {
-		return err
-	}
-	for _, disk := range disks {
-		shadowPath := filepath.Join(disk.Path, "shadow")
-		files, err := ioutil.ReadDir(shadowPath)
-		if err != nil {
-			if !os.IsNotExist(err) {
-				return fmt.Errorf("can't read '%s': %v", shadowPath, err)
-			}
-		}
-		if len(files) > 0 {
-			return fmt.Errorf("'%s' is not empty, execute 'clean' command first", shadowPath)
-		}
-	}
-	allTables, err := ch.GetTables()
-	if err != nil {
-		return fmt.Errorf("can't get tables from clickhouse: %v", err)
-	}
-	backupTables := filterTablesByPattern(allTables, tablePattern)
-	if len(backupTables) == 0 {
-		return fmt.Errorf("there are no tables in clickhouse, create something to freeze")
-	}
-	for _, table := range backupTables {
-		if table.Skip {
-			log.Infof("Skip '%s.%s'", table.Database, table.Name)
-			continue
-		}
-		if err := ch.FreezeTable(&table); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // NewBackupName - return default backup name
 func NewBackupName() string {
 	return time.Now().UTC().Format(BackupTimeFormat)
@@ -483,6 +437,13 @@ func RestoreData(cfg config.Config, backupName string, tablePattern string) erro
 	if err != nil {
 		return err
 	}
+	dstTablesMap := map[metadata.TableTitle]clickhouse.Table{}
+	for i := range chTables {
+		dstTablesMap[metadata.TableTitle{
+			Database: chTables[i].Database,
+			Table:    chTables[i].Name,
+		}] = chTables[i]
+	}
 	if len(tablesForRestore) == 0 {
 		return fmt.Errorf("backup doesn't have tables to restore")
 	}
@@ -507,7 +468,10 @@ func RestoreData(cfg config.Config, backupName string, tablePattern string) erro
 		return err
 	}
 	for _, table := range tablesForRestore {
-		if err := ch.CopyData(backupName, table, disks); err != nil {
+		dstTableDataPaths := dstTablesMap[metadata.TableTitle{
+			Database: table.Database,
+			Table:    table.Table}].DataPaths
+		if err := ch.CopyData(backupName, table, disks, dstTableDataPaths); err != nil {
 			return fmt.Errorf("can't restore '%s.%s': %v", table.Database, table.Table, err)
 		}
 		if err := ch.AttachPartitions(table, disks); err != nil {
@@ -754,30 +718,6 @@ func Download(cfg config.Config, backupName string, tablePattern string, schemaO
 	return nil
 }
 
-// Clean - removed all data in shadow folder
-func Clean(cfg config.Config) error {
-	ch := &clickhouse.ClickHouse{
-		Config: &cfg.ClickHouse,
-	}
-	if err := ch.Connect(); err != nil {
-		return fmt.Errorf("can't connect to clickhouse: %v", err)
-	}
-	defer ch.Close()
-
-	disks, err := ch.GetDisks()
-	if err != nil {
-		return err
-	}
-	for _, disk := range disks {
-		shadowDir := path.Join(disk.Path, "shadow")
-		log.Infof("Clean %s", shadowDir)
-		if err := cleanDir(shadowDir); err != nil {
-			return fmt.Errorf("can't clean '%s': %v", shadowDir, err)
-		}
-	}
-	return nil
-}
-
 //
 func RemoveOldBackupsLocal(cfg config.Config) error {
 	if cfg.General.BackupsToKeepLocal < 1 {

pkg/backup/print.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"text/tabwriter"
 
 	"github.com/AlexAkulov/clickhouse-backup/config"
 	"github.com/AlexAkulov/clickhouse-backup/pkg/clickhouse"
@@ -14,7 +15,7 @@ import (
 	"github.com/AlexAkulov/clickhouse-backup/utils"
 )
 
-func printBackups(backupList []new_storage.Backup, format string, printSize bool) error {
+func printBackups(backupList []new_storage.Backup, format, location string) error {
 	switch format {
 	case "latest", "last", "l":
 		if len(backupList) < 1 {
@@ -27,16 +28,14 @@ func printBackups(backupList []new_storage.Backup, format string, printSize bool
 		}
 		fmt.Println(backupList[len(backupList)-2].BackupName)
 	case "all", "":
-		if len(backupList) == 0 {
-			fmt.Println("no backups found")
-		}
+		// if len(backupList) == 0 {
+		// 	fmt.Println("no backups found")
+		// }
+		w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns)
 		for _, backup := range backupList {
-			if printSize {
-				fmt.Printf("- '%s'\t%s\t(created at %s)\n", backup.BackupName, utils.FormatBytes(backup.Size), backup.CreationDate.Format("02-01-2006 15:04:05"))
-			} else {
-				fmt.Printf("- '%s'\t(created at %s)\n", backup.BackupName, backup.CreationDate.Format("02-01-2006 15:04:05"))
-			}
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", backup.BackupName, utils.FormatBytes(backup.Size), backup.CreationDate.Format("02-01-2006 15:04:05"), location)
 		}
+		w.Flush()
 	default:
 		return fmt.Errorf("'%s' undefined", format)
 	}
@@ -49,7 +48,7 @@ func PrintLocalBackups(cfg config.Config, format string) error {
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
-	return printBackups(backupList, format, false)
+	return printBackups(backupList, format, "local")
 }
 
 // ListLocalBackups - return slice of all backups stored locally
@@ -107,7 +106,7 @@ func PrintRemoteBackups(cfg config.Config, format string) error {
 	if err != nil {
 		return err
 	}
-	return printBackups(backupList, format, true)
+	return printBackups(backupList, format, "remote")
 }
 
 func GetLocalBackup(cfg config.Config, backupName string) error {
@@ -183,6 +182,7 @@ func PrintTables(cfg config.Config, printAll bool) error {
 	if err != nil {
 		return err
 	}
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns)
 	for _, table := range allTables {
 		if table.Skip && !printAll {
 			continue
@@ -192,10 +192,11 @@ func PrintTables(cfg config.Config, printAll bool) error {
 			tableDisks = append(tableDisks, disk)
 		}
 		if table.Skip {
-			fmt.Printf("skip\t%s.%s\t%s\t%v\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes.Int64), strings.Join(tableDisks, ","))
+			fmt.Fprintf(w, "%s.%s\t%s\t%v\tskip\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes.Int64), strings.Join(tableDisks, ","))
 			continue
 		}
-		fmt.Printf("%s.%s\t%s\t%v\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes.Int64), strings.Join(tableDisks, ","))
+		fmt.Fprintf(w, "%s.%s\t%s\t%v\t\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes.Int64), strings.Join(tableDisks, ","))
 	}
+	w.Flush()
 	return nil
 }

pkg/clickhouse/clickhouse.go

@@ -199,18 +199,42 @@ func (ch *ClickHouse) GetTables() ([]Table, error) {
 		}
 		tables[i] = ch.fixVariousVersions(t)
 	}
+	if len(tables) == 0 {
+		return tables, nil
+	}
+	if !tables[0].TotalBytes.Valid {
+		tables = ch.getTableSizeFromParts(tables)
+	}
 	return tables, nil
 }
 
-// GetTable - return table
-func (ch *ClickHouse) GetTable(database, name string) (*Table, error) {
-	tables := make([]Table, 0)
-	err := ch.softSelect(&tables, fmt.Sprintf("SELECT * FROM system.tables WHERE is_temporary = 0 AND database = '%s' AND name = '%s';", database, name))
-	if err != nil {
-		return nil, err
+func (ch *ClickHouse) getTableSizeFromParts(tables []Table) []Table {
+	var tablesSize []struct {
+		Database string `db:"database"`
+		Table    string `db:"table"`
+		Size     int64  `db:"size"`
+	}
+	ch.softSelect(&tablesSize, fmt.Sprintf("SELECT database, table, sum(bytes_on_disk) as size FROM system.parts GROUP BY (database, table)"))
+	tableMap := map[metadata.TableTitle]int64{}
+	for i := range tablesSize {
+		tableMap[metadata.TableTitle{
+			Database: tablesSize[i].Database,
+			Table:    tablesSize[i].Table,
+		}] = tablesSize[i].Size
+	}
+	for i, t := range tables {
+		if t.TotalBytes.Valid {
+			continue
+		}
+		t.TotalBytes = sql.NullInt64{
+			Int64: tableMap[metadata.TableTitle{
+				Database: t.Database,
+				Table:    t.Name}],
+			Valid: true,
+		}
+		tables[i] = t
 	}
-	result := ch.fixVariousVersions(tables[0])
-	return &result, nil
+	return tables
 }
 
 func (ch *ClickHouse) fixVariousVersions(t Table) Table {
@@ -334,13 +358,9 @@ func (ch *ClickHouse) Mkdir(name string) error {
 }
 
 // CopyData - copy partitions for specific table to detached folder
-func (ch *ClickHouse) CopyData(backupName string, backupTable metadata.TableMetadata, disks []Disk) error {
+func (ch *ClickHouse) CopyData(backupName string, backupTable metadata.TableMetadata, disks []Disk, tableDataPaths []string) error {
 	// TODO: проверить если диск есть в бэкапе но нет в кликхаусе
-	dstTable, err := ch.GetTable(backupTable.Database, backupTable.Table)
-	if err != nil {
-		return err
-	}
-	dstDataPaths := GetDisksByPaths(disks, dstTable.DataPaths)
+	dstDataPaths := GetDisksByPaths(disks, tableDataPaths)
 	for _, backupDisk := range disks {
 		if len(backupTable.Parts[backupDisk.Name]) == 0 {
 			continue