From a467f512cb6b2ea0c2afa45fc99d49aaefc88926 Mon Sep 17 00:00:00 2001 From: David Kohn Date: Fri, 13 Nov 2020 10:46:55 -0500 Subject: [PATCH] Automatically update TimescaleDB version after restore Adds a flag and the ability to update to the latest default version after a restore. The default version is the one that is chosen when you run `ALTER EXTENSION timescaledb UPDATE;` without a version flag. --- README.md | 11 ++++++++-- cmd/ts-restore/main.go | 2 +- pkg/restore/restore.go | 43 ++++++++++++++++++++++++++++-------- pkg/test/restore_test.go | 47 +++++++++++++++++++++++++++------------- pkg/util/util.go | 1 + 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 2806f7f..de0a500 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,8 @@ version, we recommend restoring only to an empty database. You will need to pro Optional parameters: - `--jobs` Sets the number of jobs to run for the restore, by default it is set to 4 and will run in parallel mode during the sections[^1] that are able to be parallelized. Set to 0 to disable parallelism. - - `--verbose` Determines whether verbose output will be provided from `pg_restore`. Defaults to true. + - `--verbose` Provide verbose output from `pg_restore`. Defaults to true. + - `--do-update` Update the TimescaleDB version to the latest default version immediately following the restore.[^2] Defaults to true. As an example, let's suppose I have two `postgres` clusters running on my machine, perhaps on versions 11 on port 5432 and 12 on 5433 and I wish to dump and restore in order to upgrade between versions: I would run `ts-dump --db-URI=postgresql://postgres:pwd1@localhost:5432/tsdb --dump-dir=~/dumps/dump1 --verbose --jobs=2` @@ -91,4 +92,10 @@ include. [^1]: In order to support parallel restores given the way the TimescaleDB catalog works, we first perform a pre-data restore, then restore the data for the catalog, then in parallel perform the data section for everything else and the post-data section for - everything. \ No newline at end of file + everything. + +[^2]: This requires that you have the .so for the version you are restoring from and the + version that you will update to. For instance, if your dump is from TimescaleDB 1.6.2 + and the latest version is TimescaleDB 1.7.4, you need the .so from 1.6.2 available to + restore to, and then it will update to 1.7.4 following the restore. Our default + packages include several older versions to enable updates. \ No newline at end of file diff --git a/cmd/ts-restore/main.go b/cmd/ts-restore/main.go index d8ba421..c470210 100644 --- a/cmd/ts-restore/main.go +++ b/cmd/ts-restore/main.go @@ -16,7 +16,7 @@ func main() { config = util.RegisterCommonConfigFlags(config) // for restore we want to default to verbose output, it gives good information about how the restore is proceeding flag.BoolVar(&config.Verbose, "verbose", true, "specifies whether verbose output is requested, default true") - + flag.BoolVar(&config.DoUpdate, "do-update", true, "set to false to leave TimescaleDB at the dumped version, defaults to true, which upgrades to default installed") flag.Parse() config, err := util.CleanConfig(config) if err != nil { diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 57f780a..8f4afc3 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -13,7 +13,6 @@ import ( "os" "os/exec" - "github.com/jackc/pgx/v4" "github.com/timescale/ts-dump-restore/pkg/util" ) @@ -95,7 +94,16 @@ func DoRestore(cf *util.Config) error { if err != nil { return fmt.Errorf("pg_restore run failed during post-data step: %w", err) } + + //Now perform the extension update if we're doing that. + if cf.DoUpdate { + err = doUpdate(cf.DbURI) + if err != nil { + return fmt.Errorf("pg_restore run failed while updating extension: %w", err) + } + } return err + } func getRestoreCmd(restorePath string, dumpDir string, baseArgs []string, addlArgs ...string) *exec.Cmd { @@ -202,18 +210,35 @@ func postRestoreTimescale(dbURI string, tsInfo util.TsInfo) error { if !pr { return errors.New("post restore function failed") } + return err +} + +func doUpdate(dbURI string) error { - // Confirm that no one pulled a fast one on us, and that we're at the right extension version - info := util.TsInfo{} - err = conn.QueryRow(context.Background(), "SELECT e.extversion, n.nspname FROM pg_extension e INNER JOIN pg_namespace n ON e.extnamespace = n.oid WHERE e.extname='timescaledb'").Scan(&info.TsVersion, &info.TsSchema) + conn, err := util.GetDBConn(context.Background(), dbURI) + if err != nil { + return err + } + defer conn.Close(context.Background()) + _, err = conn.Exec(context.Background(), "ALTER EXTENSION timescaledb UPDATE ") + if err != nil { + return fmt.Errorf("failed to update extension version: %w", err) + } + conn.Close(context.Background()) // close the alter extension connection + conn2, err := util.GetDBConn(context.Background(), dbURI) // open a new one to confirm we can make a connection + if err != nil { + return fmt.Errorf("failed to connect after updating extension:%w", err) + } + defer conn2.Close(context.Background()) + + // confirm that the installed version now matches the default version + var vm bool + err = conn2.QueryRow(context.Background(), "SELECT installed_version = default_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'").Scan(&vm) if err != nil { - if err == pgx.ErrNoRows { - return errors.New("could not confirm creation of TimescaleDB extension") - } return err } - if info.TsSchema != tsInfo.TsSchema || info.TsVersion != tsInfo.TsVersion { - return errors.New("TimescaleDB extension created in incorrect schema or at incorrect version, please drop the extension and restart the restore") + if !vm { + return errors.New("TimescaleDB extension was not updated to the default version") } return err } diff --git a/pkg/test/restore_test.go b/pkg/test/restore_test.go index b5099b2..b21d0e5 100644 --- a/pkg/test/restore_test.go +++ b/pkg/test/restore_test.go @@ -64,55 +64,71 @@ func TestBackupRestore(t *testing.T) { restoreImage string tsVersion string numJobs int + doUpdate bool }{ { desc: "pg-11-parallel", - dumpImage: "timescale/timescaledb:latest-pg11", - restoreImage: "timescale/timescaledb:latest-pg11", + dumpImage: "timescale/timescaledb:1.7.2-pg11", + restoreImage: "timescale/timescaledb:1.7.2-pg11", tsVersion: "1.7.1", numJobs: 4, + doUpdate: false, }, { desc: "pg-11-non-parallel", - dumpImage: "timescale/timescaledb:latest-pg11", - restoreImage: "timescale/timescaledb:latest-pg11", + dumpImage: "timescale/timescaledb:1.7.2-pg11", + restoreImage: "timescale/timescaledb:1.7.2-pg11", tsVersion: "1.7.1", numJobs: 0, + doUpdate: false, }, { desc: "pg-12-parallel", - dumpImage: "timescale/timescaledb:latest-pg12", - restoreImage: "timescale/timescaledb:latest-pg12", + dumpImage: "timescale/timescaledb:1.7.2-pg12", + restoreImage: "timescale/timescaledb:1.7.2-pg12", tsVersion: "1.7.1", numJobs: 4, + doUpdate: false, }, { desc: "pg-11-to-12", - dumpImage: "timescale/timescaledb:latest-pg11", - restoreImage: "timescale/timescaledb:latest-pg12", + dumpImage: "timescale/timescaledb:1.7.2-pg11", + restoreImage: "timescale/timescaledb:1.7.2-pg12", tsVersion: "1.7.1", numJobs: 4, + doUpdate: false, }, { desc: "pg-11-older-ts-1.6.1", - dumpImage: "timescale/timescaledb:latest-pg11", - restoreImage: "timescale/timescaledb:latest-pg11", + dumpImage: "timescale/timescaledb:1.7.2-pg11", + restoreImage: "timescale/timescaledb:1.7.2-pg11", tsVersion: "1.6.1", numJobs: 4, + doUpdate: false, }, { desc: "pg-10-parallel", - dumpImage: "timescale/timescaledb:latest-pg10", - restoreImage: "timescale/timescaledb:latest-pg10", - tsVersion: "1.7.1", + dumpImage: "timescale/timescaledb:1.7.2-pg10", + restoreImage: "timescale/timescaledb:1.7.2-pg10", + tsVersion: "1.7.2", numJobs: 4, + doUpdate: false, }, { desc: "pg-10-11-upgrade-ts-1.6.1", - dumpImage: "timescale/timescaledb:latest-pg10", - restoreImage: "timescale/timescaledb:latest-pg11", + dumpImage: "timescale/timescaledb:1.7.2-pg10", + restoreImage: "timescale/timescaledb:1.7.2-pg11", tsVersion: "1.6.1", numJobs: 4, + doUpdate: false, + }, + { + desc: "pg-12-update", + dumpImage: "timescale/timescaledb:1.7.4-pg12", + restoreImage: "timescale/timescaledb:1.7.4-pg12", + tsVersion: "1.7.1", + numJobs: 4, + doUpdate: true, }, } for _, c := range cases { @@ -148,6 +164,7 @@ func TestBackupRestore(t *testing.T) { restoreConfig.DumpDir = fmt.Sprintf("%s.%d", dumpDb.dbName, dumpDb.port.Int()) //dump dir is from dump not from restore restoreConfig.Verbose = true //default settings restoreConfig.Jobs = c.numJobs + restoreConfig.DoUpdate = c.doUpdate util.CleanConfig(restoreConfig) //make sure we remove the dumpDir at the end no matter what diff --git a/pkg/util/util.go b/pkg/util/util.go index 4142e58..6a3e8d2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -27,6 +27,7 @@ type Config struct { TsInfoFileName string Verbose bool Jobs int + DoUpdate bool // whether to do an update after restoring. } //TsInfo holds information about the Timescale installation