diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 491491592f58..08bd8d53c949 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -38,7 +38,7 @@ - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb - dockerImageTag: 1.0.17 + dockerImageTag: 1.0.32 documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb icon: alloydb.svg sourceType: database @@ -1250,7 +1250,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.31 + dockerImageTag: 1.0.32 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9341817ecf6a..36e887c0dc2e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -299,7 +299,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-alloydb:1.0.17" +- dockerImage: "airbyte/source-alloydb:1.0.32" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -389,7 +389,7 @@ order: 7 oneOf: - title: "disable" - additionalProperties: false + additionalProperties: true description: "Disable SSL." required: - "mode" @@ -397,12 +397,9 @@ mode: type: "string" const: "disable" - enum: - - "disable" - default: "disable" order: 0 - title: "allow" - additionalProperties: false + additionalProperties: true description: "Allow SSL mode." required: - "mode" @@ -410,12 +407,9 @@ mode: type: "string" const: "allow" - enum: - - "allow" - default: "allow" order: 0 - title: "prefer" - additionalProperties: false + additionalProperties: true description: "Prefer SSL mode." required: - "mode" @@ -423,12 +417,9 @@ mode: type: "string" const: "prefer" - enum: - - "prefer" - default: "prefer" order: 0 - title: "require" - additionalProperties: false + additionalProperties: true description: "Require SSL mode." required: - "mode" @@ -436,12 +427,9 @@ mode: type: "string" const: "require" - enum: - - "require" - default: "require" order: 0 - title: "verify-ca" - additionalProperties: false + additionalProperties: true description: "Verify-ca SSL mode." required: - "mode" @@ -450,9 +438,6 @@ mode: type: "string" const: "verify-ca" - enum: - - "verify-ca" - default: "verify-ca" order: 0 ca_certificate: type: "string" @@ -483,7 +468,7 @@ airbyte_secret: true order: 4 - title: "verify-full" - additionalProperties: false + additionalProperties: true description: "Verify-full SSL mode." required: - "mode" @@ -492,9 +477,6 @@ mode: type: "string" const: "verify-full" - enum: - - "verify-full" - default: "verify-full" order: 0 ca_certificate: type: "string" @@ -539,9 +521,6 @@ method: type: "string" const: "Standard" - enum: - - "Standard" - default: "Standard" order: 0 - title: "Logical Replication (CDC)" description: "Logical replication uses the Postgres write-ahead log (WAL)\ @@ -557,10 +536,7 @@ method: type: "string" const: "CDC" - enum: - - "CDC" - default: "CDC" - order: 0 + order: 1 plugin: type: "string" title: "Plugin" @@ -573,21 +549,21 @@ - "pgoutput" - "wal2json" default: "pgoutput" - order: 1 + order: 2 replication_slot: type: "string" title: "Replication Slot" description: "A plugin logical replication slot. Read about replication slots." - order: 2 + order: 3 publication: type: "string" title: "Publication" description: "A Postgres publication used for consuming changes. Read\ \ about publications and replication identities." - order: 3 + order: 4 initial_waiting_seconds: type: "integer" title: "Initial Waiting Time in Seconds (Advanced)" @@ -597,7 +573,7 @@ \ initial waiting time." default: 300 - order: 4 + order: 5 min: 120 max: 1200 tunnel_method: @@ -11346,7 +11322,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.31" +- dockerImage: "airbyte/source-postgres:1.0.32" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index 1254b49f6f25..66c0e5c4187e 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.17 +LABEL io.airbyte.version=1.0.32 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index f7f3f77372ac..aab298f11a05 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.17 +LABEL io.airbyte.version=1.0.32 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index c7cd569afcb9..3da938aea111 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.31 +LABEL io.airbyte.version=1.0.32 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index d209143dd0e9..7032d938e5c3 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.31 +LABEL io.airbyte.version=1.0.32 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 8faef1fa162b..ef6d2d354889 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -51,6 +51,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -545,6 +546,7 @@ private JsonNode buildConfigEscapingNeeded() { } @Test + @Disabled("See https://github.com/airbytehq/airbyte/issues/17150#issuecomment-1342898439, enable once communication is out") public void tableWithNullValueCursorShouldThrowException() throws SQLException { try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { db.start(); @@ -586,6 +588,7 @@ private ConfiguredAirbyteStream createTableWithNullValueCursor(final Database da } @Test + @Disabled("See https://github.com/airbytehq/airbyte/issues/17150#issuecomment-1342898439, enable once communication is out") public void viewWithNullValueCursorShouldThrowException() throws SQLException { try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { db.start(); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java index 2df3f4b6620f..595e55da2c80 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java @@ -184,6 +184,7 @@ private void validateCursorFieldForIncrementalTables( final Map>> tableNameToTable, final ConfiguredAirbyteCatalog catalog, final Database database) throws SQLException { final List tablesWithInvalidCursor = new ArrayList<>(); + final List tablesWithInvalidCursorToWarnAbout = new ArrayList<>(); for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { final AirbyteStream stream = airbyteStream.getStream(); final String fullyQualifiedTableName = getFullyQualifiedTableName(stream.getNamespace(), @@ -216,13 +217,16 @@ private void validateCursorFieldForIncrementalTables( } if (!verifyCursorColumnValues(database, stream.getNamespace(), stream.getName(), cursorField.get())) { - tablesWithInvalidCursor.add( + tablesWithInvalidCursorToWarnAbout.add( new InvalidCursorInfo(fullyQualifiedTableName, cursorField.get(), cursorType.toString(), "Cursor column contains NULL value")); - continue; } } + if (!tablesWithInvalidCursorToWarnAbout.isEmpty()) { + LOGGER.warn("source-postgres detected null cursor value " + InvalidCursorInfoUtil.getInvalidCursorConfigMessage(tablesWithInvalidCursor)); + } + if (!tablesWithInvalidCursor.isEmpty()) { throw new ConfigErrorException( InvalidCursorInfoUtil.getInvalidCursorConfigMessage(tablesWithInvalidCursor)); diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index c997cdf3d67e..dd2468aae6a2 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -415,6 +415,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | | 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | | | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | | 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. |