Skip to content
This repository has been archived by the owner on Feb 19, 2022. It is now read-only.

Commit

Permalink
Automatically update TimescaleDB version after restore
Browse files Browse the repository at this point in the history
Adds a flag and the ability to update to the latest default version
after a restore. The default version is the one that is chosen when
you run `ALTER EXTENSION timescaledb UPDATE;` without a version flag.
  • Loading branch information
davidkohn88 committed Nov 13, 2020
1 parent 1c88a50 commit a467f51
Show file tree
Hide file tree
Showing 5 changed files with 77 additions and 27 deletions.
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ version, we recommend restoring only to an empty database. You will need to pro

Optional parameters:
- `--jobs` Sets the number of jobs to run for the restore, by default it is set to 4 and will run in parallel mode during the sections[^1] that are able to be parallelized. Set to 0 to disable parallelism.
- `--verbose` Determines whether verbose output will be provided from `pg_restore`. Defaults to true.
- `--verbose` Provide verbose output from `pg_restore`. Defaults to true.
- `--do-update` Update the TimescaleDB version to the latest default version immediately following the restore.[^2] Defaults to true.

As an example, let's suppose I have two `postgres` clusters running on my machine, perhaps on versions 11 on port 5432 and 12 on 5433 and I wish to dump and restore in order to upgrade between versions:
I would run `ts-dump --db-URI=postgresql://postgres:pwd1@localhost:5432/tsdb --dump-dir=~/dumps/dump1 --verbose --jobs=2`
Expand All @@ -91,4 +92,10 @@ include.
[^1]: In order to support parallel restores given the way the TimescaleDB catalog works,
we first perform a pre-data restore, then restore the data for the catalog, then in
parallel perform the data section for everything else and the post-data section for
everything.
everything.

[^2]: This requires that you have the .so for the version you are restoring from and the
version that you will update to. For instance, if your dump is from TimescaleDB 1.6.2
and the latest version is TimescaleDB 1.7.4, you need the .so from 1.6.2 available to
restore to, and then it will update to 1.7.4 following the restore. Our default
packages include several older versions to enable updates.
2 changes: 1 addition & 1 deletion cmd/ts-restore/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func main() {
config = util.RegisterCommonConfigFlags(config)
// for restore we want to default to verbose output, it gives good information about how the restore is proceeding
flag.BoolVar(&config.Verbose, "verbose", true, "specifies whether verbose output is requested, default true")

flag.BoolVar(&config.DoUpdate, "do-update", true, "set to false to leave TimescaleDB at the dumped version, defaults to true, which upgrades to default installed")
flag.Parse()
config, err := util.CleanConfig(config)
if err != nil {
Expand Down
43 changes: 34 additions & 9 deletions pkg/restore/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"os"
"os/exec"

"github.com/jackc/pgx/v4"
"github.com/timescale/ts-dump-restore/pkg/util"
)

Expand Down Expand Up @@ -95,7 +94,16 @@ func DoRestore(cf *util.Config) error {
if err != nil {
return fmt.Errorf("pg_restore run failed during post-data step: %w", err)
}

//Now perform the extension update if we're doing that.
if cf.DoUpdate {
err = doUpdate(cf.DbURI)
if err != nil {
return fmt.Errorf("pg_restore run failed while updating extension: %w", err)
}
}
return err

}

func getRestoreCmd(restorePath string, dumpDir string, baseArgs []string, addlArgs ...string) *exec.Cmd {
Expand Down Expand Up @@ -202,18 +210,35 @@ func postRestoreTimescale(dbURI string, tsInfo util.TsInfo) error {
if !pr {
return errors.New("post restore function failed")
}
return err
}

func doUpdate(dbURI string) error {

// Confirm that no one pulled a fast one on us, and that we're at the right extension version
info := util.TsInfo{}
err = conn.QueryRow(context.Background(), "SELECT e.extversion, n.nspname FROM pg_extension e INNER JOIN pg_namespace n ON e.extnamespace = n.oid WHERE e.extname='timescaledb'").Scan(&info.TsVersion, &info.TsSchema)
conn, err := util.GetDBConn(context.Background(), dbURI)
if err != nil {
return err
}
defer conn.Close(context.Background())
_, err = conn.Exec(context.Background(), "ALTER EXTENSION timescaledb UPDATE ")
if err != nil {
return fmt.Errorf("failed to update extension version: %w", err)
}
conn.Close(context.Background()) // close the alter extension connection
conn2, err := util.GetDBConn(context.Background(), dbURI) // open a new one to confirm we can make a connection
if err != nil {
return fmt.Errorf("failed to connect after updating extension:%w", err)
}
defer conn2.Close(context.Background())

// confirm that the installed version now matches the default version
var vm bool
err = conn2.QueryRow(context.Background(), "SELECT installed_version = default_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'").Scan(&vm)
if err != nil {
if err == pgx.ErrNoRows {
return errors.New("could not confirm creation of TimescaleDB extension")
}
return err
}
if info.TsSchema != tsInfo.TsSchema || info.TsVersion != tsInfo.TsVersion {
return errors.New("TimescaleDB extension created in incorrect schema or at incorrect version, please drop the extension and restart the restore")
if !vm {
return errors.New("TimescaleDB extension was not updated to the default version")
}
return err
}
47 changes: 32 additions & 15 deletions pkg/test/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,55 +64,71 @@ func TestBackupRestore(t *testing.T) {
restoreImage string
tsVersion string
numJobs int
doUpdate bool
}{
{
desc: "pg-11-parallel",
dumpImage: "timescale/timescaledb:latest-pg11",
restoreImage: "timescale/timescaledb:latest-pg11",
dumpImage: "timescale/timescaledb:1.7.2-pg11",
restoreImage: "timescale/timescaledb:1.7.2-pg11",
tsVersion: "1.7.1",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-11-non-parallel",
dumpImage: "timescale/timescaledb:latest-pg11",
restoreImage: "timescale/timescaledb:latest-pg11",
dumpImage: "timescale/timescaledb:1.7.2-pg11",
restoreImage: "timescale/timescaledb:1.7.2-pg11",
tsVersion: "1.7.1",
numJobs: 0,
doUpdate: false,
},
{
desc: "pg-12-parallel",
dumpImage: "timescale/timescaledb:latest-pg12",
restoreImage: "timescale/timescaledb:latest-pg12",
dumpImage: "timescale/timescaledb:1.7.2-pg12",
restoreImage: "timescale/timescaledb:1.7.2-pg12",
tsVersion: "1.7.1",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-11-to-12",
dumpImage: "timescale/timescaledb:latest-pg11",
restoreImage: "timescale/timescaledb:latest-pg12",
dumpImage: "timescale/timescaledb:1.7.2-pg11",
restoreImage: "timescale/timescaledb:1.7.2-pg12",
tsVersion: "1.7.1",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-11-older-ts-1.6.1",
dumpImage: "timescale/timescaledb:latest-pg11",
restoreImage: "timescale/timescaledb:latest-pg11",
dumpImage: "timescale/timescaledb:1.7.2-pg11",
restoreImage: "timescale/timescaledb:1.7.2-pg11",
tsVersion: "1.6.1",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-10-parallel",
dumpImage: "timescale/timescaledb:latest-pg10",
restoreImage: "timescale/timescaledb:latest-pg10",
tsVersion: "1.7.1",
dumpImage: "timescale/timescaledb:1.7.2-pg10",
restoreImage: "timescale/timescaledb:1.7.2-pg10",
tsVersion: "1.7.2",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-10-11-upgrade-ts-1.6.1",
dumpImage: "timescale/timescaledb:latest-pg10",
restoreImage: "timescale/timescaledb:latest-pg11",
dumpImage: "timescale/timescaledb:1.7.2-pg10",
restoreImage: "timescale/timescaledb:1.7.2-pg11",
tsVersion: "1.6.1",
numJobs: 4,
doUpdate: false,
},
{
desc: "pg-12-update",
dumpImage: "timescale/timescaledb:1.7.4-pg12",
restoreImage: "timescale/timescaledb:1.7.4-pg12",
tsVersion: "1.7.1",
numJobs: 4,
doUpdate: true,
},
}
for _, c := range cases {
Expand Down Expand Up @@ -148,6 +164,7 @@ func TestBackupRestore(t *testing.T) {
restoreConfig.DumpDir = fmt.Sprintf("%s.%d", dumpDb.dbName, dumpDb.port.Int()) //dump dir is from dump not from restore
restoreConfig.Verbose = true //default settings
restoreConfig.Jobs = c.numJobs
restoreConfig.DoUpdate = c.doUpdate
util.CleanConfig(restoreConfig)

//make sure we remove the dumpDir at the end no matter what
Expand Down
1 change: 1 addition & 0 deletions pkg/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Config struct {
TsInfoFileName string
Verbose bool
Jobs int
DoUpdate bool // whether to do an update after restoring.
}

//TsInfo holds information about the Timescale installation
Expand Down

0 comments on commit a467f51

Please sign in to comment.