Merge pull request #26 from timescale/update-extension

Automatically update TimescaleDB version after restore

README.md

@@ -73,7 +73,8 @@ version, we recommend restoring only to an empty database.  You will need to pro
 
 Optional parameters:
    - `--jobs` Sets the number of jobs to run for the restore, by default it is set to 4 and will run in parallel mode during the sections[^1] that are able to be parallelized. Set to 0 to disable parallelism.
-   - `--verbose` Determines whether verbose output will be provided from `pg_restore`. Defaults to true.
+   - `--verbose` Provide verbose output from `pg_restore`. Defaults to true.
+   - `--do-update` Update the TimescaleDB version to the latest default version immediately following the restore.[^2] Defaults to true.
 
 As an example, let's suppose I have two `postgres` clusters running on my machine, perhaps on versions 11 on port 5432 and 12 on 5433 and I wish to dump and restore in order to upgrade between versions: 
 I would run `ts-dump --db-URI=postgresql://postgres:pwd1@localhost:5432/tsdb --dump-dir=~/dumps/dump1 --verbose --jobs=2`
@@ -91,4 +92,10 @@ include.
 [^1]: In order to support parallel restores given the way the TimescaleDB catalog works,
    we first perform a pre-data restore, then restore the data for the catalog, then in
    parallel perform the data section for everything else and the post-data section for
-   everything. 
+   everything. 
+
+[^2]: This requires that you have the .so for the version you are restoring from and the
+   version that you will update to. For instance, if your dump is from TimescaleDB 1.6.2
+   and the latest version is TimescaleDB 1.7.4, you need the .so from 1.6.2 available to
+   restore to, and then it will update to 1.7.4 following the restore. Our default
+   packages include several older versions to enable updates. 

cmd/ts-restore/main.go

@@ -16,7 +16,7 @@ func main() {
 	config = util.RegisterCommonConfigFlags(config)
 	// for restore we want to default to verbose output, it gives good information about how the restore is proceeding
 	flag.BoolVar(&config.Verbose, "verbose", true, "specifies whether verbose output is requested, default true")
-
+	flag.BoolVar(&config.DoUpdate, "do-update", true, "set to false to leave TimescaleDB at the dumped version, defaults to true, which upgrades to default installed")
 	flag.Parse()
 	config, err := util.CleanConfig(config)
 	if err != nil {

pkg/restore/restore.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"os/exec"
 
-	"github.com/jackc/pgx/v4"
 	"github.com/timescale/ts-dump-restore/pkg/util"
 )
 
@@ -95,7 +94,16 @@ func DoRestore(cf *util.Config) error {
 	if err != nil {
 		return fmt.Errorf("pg_restore run failed during post-data step: %w", err)
 	}
+
+	//Now perform the extension update if we're doing that.
+	if cf.DoUpdate {
+		err = doUpdate(cf.DbURI)
+		if err != nil {
+			return fmt.Errorf("pg_restore run failed while updating extension: %w", err)
+		}
+	}
 	return err
+
 }
 
 func getRestoreCmd(restorePath string, dumpDir string, baseArgs []string, addlArgs ...string) *exec.Cmd {
@@ -202,18 +210,35 @@ func postRestoreTimescale(dbURI string, tsInfo util.TsInfo) error {
 	if !pr {
 		return errors.New("post restore function failed")
 	}
+	return err
+}
+
+func doUpdate(dbURI string) error {
 
-	// Confirm that no one pulled a fast one on us, and that we're at the right extension version
-	info := util.TsInfo{}
-	err = conn.QueryRow(context.Background(), "SELECT e.extversion, n.nspname FROM pg_extension e INNER JOIN pg_namespace n ON e.extnamespace = n.oid WHERE e.extname='timescaledb'").Scan(&info.TsVersion, &info.TsSchema)
+	conn, err := util.GetDBConn(context.Background(), dbURI)
+	if err != nil {
+		return err
+	}
+	defer conn.Close(context.Background())
+	_, err = conn.Exec(context.Background(), "ALTER EXTENSION timescaledb UPDATE ")
+	if err != nil {
+		return fmt.Errorf("failed to update extension version: %w", err)
+	}
+	conn.Close(context.Background())                          // close the alter extension connection
+	conn2, err := util.GetDBConn(context.Background(), dbURI) // open a new one to confirm we can make a connection
+	if err != nil {
+		return fmt.Errorf("failed to connect after updating extension:%w", err)
+	}
+	defer conn2.Close(context.Background())
+
+	// confirm that the installed version now matches the default version
+	var vm bool
+	err = conn2.QueryRow(context.Background(), "SELECT installed_version = default_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'").Scan(&vm)
 	if err != nil {
-		if err == pgx.ErrNoRows {
-			return errors.New("could not confirm creation of TimescaleDB extension")
-		}
 		return err
 	}
-	if info.TsSchema != tsInfo.TsSchema || info.TsVersion != tsInfo.TsVersion {
-		return errors.New("TimescaleDB extension created in incorrect schema or at incorrect version, please drop the extension and restart the restore")
+	if !vm {
+		return errors.New("TimescaleDB extension was not updated to the default version")
 	}
 	return err
 }

pkg/test/restore_test.go

@@ -64,55 +64,71 @@ func TestBackupRestore(t *testing.T) {
 		restoreImage string
 		tsVersion    string
 		numJobs      int
+		doUpdate     bool
 	}{
 		{
 			desc:         "pg-11-parallel",
-			dumpImage:    "timescale/timescaledb:latest-pg11",
-			restoreImage: "timescale/timescaledb:latest-pg11",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg11",
+			restoreImage: "timescale/timescaledb:1.7.2-pg11",
 			tsVersion:    "1.7.1",
 			numJobs:      4,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-11-non-parallel",
-			dumpImage:    "timescale/timescaledb:latest-pg11",
-			restoreImage: "timescale/timescaledb:latest-pg11",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg11",
+			restoreImage: "timescale/timescaledb:1.7.2-pg11",
 			tsVersion:    "1.7.1",
 			numJobs:      0,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-12-parallel",
-			dumpImage:    "timescale/timescaledb:latest-pg12",
-			restoreImage: "timescale/timescaledb:latest-pg12",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg12",
+			restoreImage: "timescale/timescaledb:1.7.2-pg12",
 			tsVersion:    "1.7.1",
 			numJobs:      4,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-11-to-12",
-			dumpImage:    "timescale/timescaledb:latest-pg11",
-			restoreImage: "timescale/timescaledb:latest-pg12",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg11",
+			restoreImage: "timescale/timescaledb:1.7.2-pg12",
 			tsVersion:    "1.7.1",
 			numJobs:      4,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-11-older-ts-1.6.1",
-			dumpImage:    "timescale/timescaledb:latest-pg11",
-			restoreImage: "timescale/timescaledb:latest-pg11",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg11",
+			restoreImage: "timescale/timescaledb:1.7.2-pg11",
 			tsVersion:    "1.6.1",
 			numJobs:      4,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-10-parallel",
-			dumpImage:    "timescale/timescaledb:latest-pg10",
-			restoreImage: "timescale/timescaledb:latest-pg10",
-			tsVersion:    "1.7.1",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg10",
+			restoreImage: "timescale/timescaledb:1.7.2-pg10",
+			tsVersion:    "1.7.2",
 			numJobs:      4,
+			doUpdate:     false,
 		},
 		{
 			desc:         "pg-10-11-upgrade-ts-1.6.1",
-			dumpImage:    "timescale/timescaledb:latest-pg10",
-			restoreImage: "timescale/timescaledb:latest-pg11",
+			dumpImage:    "timescale/timescaledb:1.7.2-pg10",
+			restoreImage: "timescale/timescaledb:1.7.2-pg11",
 			tsVersion:    "1.6.1",
 			numJobs:      4,
+			doUpdate:     false,
+		},
+		{
+			desc:         "pg-12-update",
+			dumpImage:    "timescale/timescaledb:1.7.4-pg12",
+			restoreImage: "timescale/timescaledb:1.7.4-pg12",
+			tsVersion:    "1.7.1",
+			numJobs:      4,
+			doUpdate:     true,
 		},
 	}
 	for _, c := range cases {
@@ -148,6 +164,7 @@ func TestBackupRestore(t *testing.T) {
 			restoreConfig.DumpDir = fmt.Sprintf("%s.%d", dumpDb.dbName, dumpDb.port.Int()) //dump dir is from dump not from restore
 			restoreConfig.Verbose = true                                                   //default settings
 			restoreConfig.Jobs = c.numJobs
+			restoreConfig.DoUpdate = c.doUpdate
 			util.CleanConfig(restoreConfig)
 
 			//make sure we remove the dumpDir at the end no matter what

pkg/util/util.go

@@ -27,6 +27,7 @@ type Config struct {
 	TsInfoFileName string
 	Verbose        bool
 	Jobs           int
+	DoUpdate       bool // whether to do an update after restoring.
 }
 
 //TsInfo holds information about the Timescale installation