feat: Postgresql17 support

* Revised TOC Reader and Writer
* Added PG17 into integration tests
* Implemented integration tests more verbose
* Fixed --no-owner and --schema-only flags definition

cmd/greenmask/cmd/dump/dump.go

@@ -102,8 +102,8 @@ func init() {
 		&Config.Dump.PgDumpOptions.ExcludeSchema, "exclude-schema", "N", []string{},
 		"dump the specified schema(s) only",
 	)
-	Cmd.Flags().StringP("no-owner", "O", "", "skip restoration of object ownership in plain-text format")
-	Cmd.Flags().StringP("schema-only", "s", "", "dump only the schema, no data")
+	Cmd.Flags().BoolP("no-owner", "O", false, "skip restoration of object ownership in plain-text format")
+	Cmd.Flags().BoolP("schema-only", "s", false, "dump only the schema, no data")
 	Cmd.Flags().StringP("superuser", "S", "", "superuser user name to use in plain-text format")
 	Cmd.Flags().StringSliceVarP(
 		&Config.Dump.PgDumpOptions.Table, "table", "t", []string{}, "dump the specified table(s) only",

cmd/greenmask/cmd/restore/restore.go

@@ -147,7 +147,7 @@ func init() {
 	Cmd.Flags().StringP("use-list", "L", "", "use table of contents from this file for selecting/ordering output")
 	Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.Schema, "schema", "n", []string{}, "restore only objects in this schema")
 	Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.ExcludeSchema, "exclude-schema", "N", []string{}, "do not restore objects in this schema")
-	Cmd.Flags().StringP("no-owner", "O", "", "skip restoration of object ownership")
+	Cmd.Flags().BoolP("no-owner", "O", false, "skip restoration of object ownership")
 	Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.Function, "function", "P", []string{}, "restore named function")
 	Cmd.Flags().BoolP("schema-only", "s", false, "restore only the schema, no data")
 	Cmd.Flags().StringP("superuser", "S", "", "superuser user name to use for disabling triggers")

docker-compose-integration.yml

@@ -21,6 +21,21 @@ services:
       timeout: 5s
       retries: 2
 
+  db-17:
+    volumes:
+      - "/var/lib/postgresql/data"
+    image: postgres:17
+    ports:
+      - "54317:5432"
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: example
+    healthcheck:
+      test: [ "CMD", "psql", "-U", "postgres" ]
+      interval: 5s
+      timeout: 1s
+      retries: 3
+
   db-16:
     volumes:
       - "/var/lib/postgresql/data"
@@ -116,7 +131,7 @@ services:
       PGPASSWORD: "example"
       FILE_DUMP: "demo-small-en.zip"
       TMP_DIR: "/tmp/schema"
-      PG_VERSIONS_CHECK: "11,12,13,14,15,16"
+      PG_VERSIONS_CHECK: "11,12,13,14,15,16,17"
 #    volumes:
 #      - "/tmp/greenmask_tests:/tmp/schema"
     build:
@@ -134,12 +149,14 @@ services:
         condition: service_healthy
       db-16:
         condition: service_healthy
+      db-17:
+        condition: service_healthy
 
   greenmask:
     volumes:
       - "/tmp"
     environment:
-      PG_VERSIONS_CHECK: "11,12,13,14,15,16"
+      PG_VERSIONS_CHECK: "11,12,13,14,15,16,17"
 
       PG_USER: postgres
       PG_PASSWORD: example
@@ -155,7 +172,6 @@ services:
       STORAGE_S3_REGION: "us-east-1"
       STORAGE_S3_ACCESS_KEY_ID: "Q3AM3UQ867SPQQA43P2F"
       STORAGE_S3_SECRET_KEY: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
-      STORAGE_S3_PREFIX: "11"
     build:
       dockerfile: docker/integration/tests/Dockerfile
       context: ./

docker/greenmask/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update \
     && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
     && apt-get update \
     && apt-get install --no-install-recommends --no-install-suggests -y \
+        postgresql-client-17 \
         postgresql-client-16 \
         postgresql-client-15 \
         postgresql-client-14 \

docker/integration/tests/Dockerfile

@@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y wget gnupg2 lsb-release make \
     && echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
     && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
     && apt-get update && apt-get install --yes --no-install-recommends --no-install-suggests  \
+        postgresql-17 \
         postgresql-16 \
         postgresql-15 \
         postgresql-14 \
@@ -26,6 +27,12 @@ for pg_version in ${PG_VERSIONS_CHECK[@]}; do \n\
     export STORAGE_S3_PREFIX="${pg_version}" \n\
     export URI="host=${PG_HOST} user=${PG_USER} password=${PG_PASSWORD} dbname=${PG_DATABASE} port=${PG_PORT}" \n\
     export PG_BIN_PATH="/usr/lib/postgresql/${pg_version}/bin/" \n\
+    echo "### DEBUG ENVIRONMENT VARIABLES ###" \n\
+    echo "PGHOST: ${PG_HOST} PGUSER: ${PG_USER} PGPASSWORD: ${PG_PASSWORD} PGDATABASE: ${PG_DATABASE} PGPORT: ${PG_PORT}" \n\
+    echo "STORAGE_S3_PREFIX: ${STORAGE_S3_PREFIX}" \n\
+    echo "URI: ${URI}" \n\
+    echo "PG_BIN_PATH: ${PG_BIN_PATH}" \n\
+    echo "### RUN TESTS ###" \n\
     go test -v ./tests/integration/... \n\
     if [ $? -eq 0 ]; then \n\
         echo "### SUCCESSFUL CHECK COMPATIBILITY WITH POSTGRESQL ${pg_version} ###" \n\

internal/db/postgres/toc/entry.go

@@ -52,6 +52,7 @@ type Entry struct {
 	Dependencies []int32 /* dumpIds of objects this one depends on */
 	NDeps        int32   /* number of Dependencies */
 	FileName     *string
+	Relkind      byte /* relation kind, only for TABLE tags */
 
 	DataDumper int32 /* Routine to dump data for object */
 

internal/db/postgres/toc/reader.go

@@ -428,6 +428,16 @@ func (r *Reader) readEntries() ([]*Entry, error) {
 			entry.Tableam = tableam
 		}
 
+		if r.version >= BackupVersions["1.16"] {
+			// The relkind data stores it as int value, but according to the sources only 1 byte is used
+			// we can safely cast it to byte
+			realKindInt, err := r.readInt()
+			if err != nil {
+				return nil, fmt.Errorf("cannot read Relkind: %w", err)
+			}
+			entry.Relkind = byte(realKindInt)
+		}
+
 		owner, err := r.readStr()
 		if err != nil {
 			return nil, fmt.Errorf("cannot read Tablespace: %w", err)

internal/db/postgres/toc/utils.go

@@ -24,7 +24,7 @@ const (
 
 const InvalidOid = 0
 
-const MaxVersion = "1.15"
+const MaxVersion = "1.16"
 
 const (
 	PgCompressionNone int32 = iota
@@ -50,6 +50,7 @@ var (
 		"1.13": MakeArchiveVersion(1, 13, 0),
 		"1.14": MakeArchiveVersion(1, 14, 0),
 		"1.15": MakeArchiveVersion(1, 15, 0),
+		"1.16": MakeArchiveVersion(1, 16, 0),
 	}
 
 	BackupFormats = map[byte]string{

internal/db/postgres/toc/writer.go

@@ -229,6 +229,14 @@ func (w *Writer) writeEntries(entries []*Entry) error {
 			}
 		}
 
+		if w.version >= BackupVersions["1.16"] {
+			// The relkind data stores it as int value, but according to the sources only 1 byte is used
+			// we can safely cast it byte to int32 and write it
+			if err := w.writeInt(int32(entry.Relkind)); err != nil {
+				return fmt.Errorf("unable to write Relkind: %w", err)
+			}
+		}
+
 		if err := w.writeStr(entry.Owner); err != nil {
 			return fmt.Errorf("unable ro write Owner: %w", err)
 		}

tests/integration/greenmask/backward_compatibility_test.go

@@ -90,6 +90,7 @@ type BackwardCompatibilitySuite struct {
 }
 
 func (suite *BackwardCompatibilitySuite) SetupSuite() {
+	log.Debug().Msg("URI: " + uri)
 	suite.Require().NotEmpty(tempDir, "-tempDir non-empty flag required")
 	suite.Require().NotEmpty(pgBinPath, "-pgBinPath non-empty flag required")
 	suite.Require().NotEmpty(uri, "-uri non-empty flag required")
@@ -136,21 +137,29 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() {
 	row := suite.conn.QueryRow(context.Background(), getVersionQuery)
 	err = row.Scan(&suite.pgVersionNum)
 	suite.Require().NoError(err, "error getting pg version")
+	log.Info().Int("version", suite.pgVersionNum).Msg("got pg version")
 
 	suite.configFilePath = path.Join(suite.tmpDir, "config.yaml")
-	confFile, err := os.Create(suite.configFilePath)
-	suite.Require().NoError(err, "error creating config.yaml file")
-	defer confFile.Close()
-	err = configStr.Execute(
-		confFile,
-		map[string]any{
-			"pgBinPath":  pgBinPath,
-			"tmpDir":     suite.tmpDir,
-			"uri":        uri,
-			"storageDir": suite.storageDir,
-			"version":    suite.pgVersionNum,
-		})
-	suite.Require().NoError(err, "error encoding config into yaml")
+	func() {
+		confFile, err := os.Create(suite.configFilePath)
+		suite.Require().NoError(err, "error creating config.yaml file")
+		defer confFile.Close()
+		err = configStr.Execute(
+			confFile,
+			map[string]any{
+				"pgBinPath":  pgBinPath,
+				"tmpDir":     suite.tmpDir,
+				"uri":        uri,
+				"storageDir": suite.storageDir,
+				"version":    suite.pgVersionNum,
+			})
+		suite.Require().NoError(err, "error encoding config into yaml")
+	}()
+	// Read file and debug
+	data, err := os.ReadFile(suite.configFilePath)
+	suite.Require().NoError(err, "error reading config file")
+	log.Debug().Msg("config file content")
+	fmt.Println(string(data))
 
 }