From 194a08dc7f10d7a44e37919a706a36cfa8e9d3c6 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Mon, 7 Oct 2024 20:51:49 +0300 Subject: [PATCH] feat: Postgresql17 support * Revised TOC Reader and Writer * Added PG17 into integration tests * Implemented integration tests more verbose * Fixed --no-owner and --schema-only flags definition --- cmd/greenmask/cmd/dump/dump.go | 4 +-- cmd/greenmask/cmd/restore/restore.go | 2 +- docker-compose-integration.yml | 22 ++++++++++-- docker/greenmask/Dockerfile | 1 + docker/integration/tests/Dockerfile | 7 ++++ internal/db/postgres/toc/entry.go | 1 + internal/db/postgres/toc/reader.go | 10 ++++++ internal/db/postgres/toc/utils.go | 3 +- internal/db/postgres/toc/writer.go | 8 +++++ .../greenmask/backward_compatibility_test.go | 35 ++++++++++++------- 10 files changed, 73 insertions(+), 20 deletions(-) diff --git a/cmd/greenmask/cmd/dump/dump.go b/cmd/greenmask/cmd/dump/dump.go index 46b6d2d4..a2374d57 100644 --- a/cmd/greenmask/cmd/dump/dump.go +++ b/cmd/greenmask/cmd/dump/dump.go @@ -102,8 +102,8 @@ func init() { &Config.Dump.PgDumpOptions.ExcludeSchema, "exclude-schema", "N", []string{}, "dump the specified schema(s) only", ) - Cmd.Flags().StringP("no-owner", "O", "", "skip restoration of object ownership in plain-text format") - Cmd.Flags().StringP("schema-only", "s", "", "dump only the schema, no data") + Cmd.Flags().BoolP("no-owner", "O", false, "skip restoration of object ownership in plain-text format") + Cmd.Flags().BoolP("schema-only", "s", false, "dump only the schema, no data") Cmd.Flags().StringP("superuser", "S", "", "superuser user name to use in plain-text format") Cmd.Flags().StringSliceVarP( &Config.Dump.PgDumpOptions.Table, "table", "t", []string{}, "dump the specified table(s) only", diff --git a/cmd/greenmask/cmd/restore/restore.go b/cmd/greenmask/cmd/restore/restore.go index 4ae7ef10..d09b6dcd 100644 --- a/cmd/greenmask/cmd/restore/restore.go +++ b/cmd/greenmask/cmd/restore/restore.go @@ -147,7 +147,7 @@ func init() { Cmd.Flags().StringP("use-list", "L", "", "use table of contents from this file for selecting/ordering output") Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.Schema, "schema", "n", []string{}, "restore only objects in this schema") Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.ExcludeSchema, "exclude-schema", "N", []string{}, "do not restore objects in this schema") - Cmd.Flags().StringP("no-owner", "O", "", "skip restoration of object ownership") + Cmd.Flags().BoolP("no-owner", "O", false, "skip restoration of object ownership") Cmd.Flags().StringSliceVarP(&Config.Restore.PgRestoreOptions.Function, "function", "P", []string{}, "restore named function") Cmd.Flags().BoolP("schema-only", "s", false, "restore only the schema, no data") Cmd.Flags().StringP("superuser", "S", "", "superuser user name to use for disabling triggers") diff --git a/docker-compose-integration.yml b/docker-compose-integration.yml index 031f7b37..1d926501 100644 --- a/docker-compose-integration.yml +++ b/docker-compose-integration.yml @@ -21,6 +21,21 @@ services: timeout: 5s retries: 2 + db-17: + volumes: + - "/var/lib/postgresql/data" + image: postgres:17 + ports: + - "54317:5432" + restart: always + environment: + POSTGRES_PASSWORD: example + healthcheck: + test: [ "CMD", "psql", "-U", "postgres" ] + interval: 5s + timeout: 1s + retries: 3 + db-16: volumes: - "/var/lib/postgresql/data" @@ -116,7 +131,7 @@ services: PGPASSWORD: "example" FILE_DUMP: "demo-small-en.zip" TMP_DIR: "/tmp/schema" - PG_VERSIONS_CHECK: "11,12,13,14,15,16" + PG_VERSIONS_CHECK: "11,12,13,14,15,16,17" # volumes: # - "/tmp/greenmask_tests:/tmp/schema" build: @@ -134,12 +149,14 @@ services: condition: service_healthy db-16: condition: service_healthy + db-17: + condition: service_healthy greenmask: volumes: - "/tmp" environment: - PG_VERSIONS_CHECK: "11,12,13,14,15,16" + PG_VERSIONS_CHECK: "11,12,13,14,15,16,17" PG_USER: postgres PG_PASSWORD: example @@ -155,7 +172,6 @@ services: STORAGE_S3_REGION: "us-east-1" STORAGE_S3_ACCESS_KEY_ID: "Q3AM3UQ867SPQQA43P2F" STORAGE_S3_SECRET_KEY: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" - STORAGE_S3_PREFIX: "11" build: dockerfile: docker/integration/tests/Dockerfile context: ./ diff --git a/docker/greenmask/Dockerfile b/docker/greenmask/Dockerfile index e1d648d0..407c36d0 100644 --- a/docker/greenmask/Dockerfile +++ b/docker/greenmask/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update \ && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ + postgresql-client-17 \ postgresql-client-16 \ postgresql-client-15 \ postgresql-client-14 \ diff --git a/docker/integration/tests/Dockerfile b/docker/integration/tests/Dockerfile index 09768959..856f89d4 100644 --- a/docker/integration/tests/Dockerfile +++ b/docker/integration/tests/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y wget gnupg2 lsb-release make \ && echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update && apt-get install --yes --no-install-recommends --no-install-suggests \ + postgresql-17 \ postgresql-16 \ postgresql-15 \ postgresql-14 \ @@ -26,6 +27,12 @@ for pg_version in ${PG_VERSIONS_CHECK[@]}; do \n\ export STORAGE_S3_PREFIX="${pg_version}" \n\ export URI="host=${PG_HOST} user=${PG_USER} password=${PG_PASSWORD} dbname=${PG_DATABASE} port=${PG_PORT}" \n\ export PG_BIN_PATH="/usr/lib/postgresql/${pg_version}/bin/" \n\ + echo "### DEBUG ENVIRONMENT VARIABLES ###" \n\ + echo "PGHOST: ${PG_HOST} PGUSER: ${PG_USER} PGPASSWORD: ${PG_PASSWORD} PGDATABASE: ${PG_DATABASE} PGPORT: ${PG_PORT}" \n\ + echo "STORAGE_S3_PREFIX: ${STORAGE_S3_PREFIX}" \n\ + echo "URI: ${URI}" \n\ + echo "PG_BIN_PATH: ${PG_BIN_PATH}" \n\ + echo "### RUN TESTS ###" \n\ go test -v ./tests/integration/... \n\ if [ $? -eq 0 ]; then \n\ echo "### SUCCESSFUL CHECK COMPATIBILITY WITH POSTGRESQL ${pg_version} ###" \n\ diff --git a/internal/db/postgres/toc/entry.go b/internal/db/postgres/toc/entry.go index af902ed3..059b5950 100644 --- a/internal/db/postgres/toc/entry.go +++ b/internal/db/postgres/toc/entry.go @@ -52,6 +52,7 @@ type Entry struct { Dependencies []int32 /* dumpIds of objects this one depends on */ NDeps int32 /* number of Dependencies */ FileName *string + Relkind byte /* relation kind, only for TABLE tags */ DataDumper int32 /* Routine to dump data for object */ diff --git a/internal/db/postgres/toc/reader.go b/internal/db/postgres/toc/reader.go index 264661d0..11bd8c0a 100644 --- a/internal/db/postgres/toc/reader.go +++ b/internal/db/postgres/toc/reader.go @@ -428,6 +428,16 @@ func (r *Reader) readEntries() ([]*Entry, error) { entry.Tableam = tableam } + if r.version >= BackupVersions["1.16"] { + // The relkind data stores it as int value, but according to the sources only 1 byte is used + // we can safely cast it to byte + realKindInt, err := r.readInt() + if err != nil { + return nil, fmt.Errorf("cannot read Relkind: %w", err) + } + entry.Relkind = byte(realKindInt) + } + owner, err := r.readStr() if err != nil { return nil, fmt.Errorf("cannot read Tablespace: %w", err) diff --git a/internal/db/postgres/toc/utils.go b/internal/db/postgres/toc/utils.go index 985f56cd..2215c490 100644 --- a/internal/db/postgres/toc/utils.go +++ b/internal/db/postgres/toc/utils.go @@ -24,7 +24,7 @@ const ( const InvalidOid = 0 -const MaxVersion = "1.15" +const MaxVersion = "1.16" const ( PgCompressionNone int32 = iota @@ -50,6 +50,7 @@ var ( "1.13": MakeArchiveVersion(1, 13, 0), "1.14": MakeArchiveVersion(1, 14, 0), "1.15": MakeArchiveVersion(1, 15, 0), + "1.16": MakeArchiveVersion(1, 16, 0), } BackupFormats = map[byte]string{ diff --git a/internal/db/postgres/toc/writer.go b/internal/db/postgres/toc/writer.go index 495c2eee..1c7ca0ae 100644 --- a/internal/db/postgres/toc/writer.go +++ b/internal/db/postgres/toc/writer.go @@ -229,6 +229,14 @@ func (w *Writer) writeEntries(entries []*Entry) error { } } + if w.version >= BackupVersions["1.16"] { + // The relkind data stores it as int value, but according to the sources only 1 byte is used + // we can safely cast it byte to int32 and write it + if err := w.writeInt(int32(entry.Relkind)); err != nil { + return fmt.Errorf("unable to write Relkind: %w", err) + } + } + if err := w.writeStr(entry.Owner); err != nil { return fmt.Errorf("unable ro write Owner: %w", err) } diff --git a/tests/integration/greenmask/backward_compatibility_test.go b/tests/integration/greenmask/backward_compatibility_test.go index 88fd418a..8145bc93 100644 --- a/tests/integration/greenmask/backward_compatibility_test.go +++ b/tests/integration/greenmask/backward_compatibility_test.go @@ -90,6 +90,7 @@ type BackwardCompatibilitySuite struct { } func (suite *BackwardCompatibilitySuite) SetupSuite() { + log.Debug().Msg("URI: " + uri) suite.Require().NotEmpty(tempDir, "-tempDir non-empty flag required") suite.Require().NotEmpty(pgBinPath, "-pgBinPath non-empty flag required") suite.Require().NotEmpty(uri, "-uri non-empty flag required") @@ -136,21 +137,29 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() { row := suite.conn.QueryRow(context.Background(), getVersionQuery) err = row.Scan(&suite.pgVersionNum) suite.Require().NoError(err, "error getting pg version") + log.Info().Int("version", suite.pgVersionNum).Msg("got pg version") suite.configFilePath = path.Join(suite.tmpDir, "config.yaml") - confFile, err := os.Create(suite.configFilePath) - suite.Require().NoError(err, "error creating config.yaml file") - defer confFile.Close() - err = configStr.Execute( - confFile, - map[string]any{ - "pgBinPath": pgBinPath, - "tmpDir": suite.tmpDir, - "uri": uri, - "storageDir": suite.storageDir, - "version": suite.pgVersionNum, - }) - suite.Require().NoError(err, "error encoding config into yaml") + func() { + confFile, err := os.Create(suite.configFilePath) + suite.Require().NoError(err, "error creating config.yaml file") + defer confFile.Close() + err = configStr.Execute( + confFile, + map[string]any{ + "pgBinPath": pgBinPath, + "tmpDir": suite.tmpDir, + "uri": uri, + "storageDir": suite.storageDir, + "version": suite.pgVersionNum, + }) + suite.Require().NoError(err, "error encoding config into yaml") + }() + // Read file and debug + data, err := os.ReadFile(suite.configFilePath) + suite.Require().NoError(err, "error reading config file") + log.Debug().Msg("config file content") + fmt.Println(string(data)) }