From 47816e1d6c7819839923e9e369c4843565ccba49 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 12 May 2022 22:49:28 -0700 Subject: [PATCH 1/5] Remove host in azure username --- .../source/postgres/PostgresSource.java | 18 +++++++++++++++++- .../source/postgres/PostgresSourceTest.java | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 0a8ee9ff0b47..01dfa386c703 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -332,7 +332,7 @@ public Set getPrivilegesTableForCurrentUser(final JdbcDatabase WHERE t.grpid = m.oid AND t.userid = r.oid) AND privilege_type = 'SELECT'; """); - final String username = database.getDatabaseConfig().get("username").asText(); + final String username = getUsername(database.getDatabaseConfig()); ps.setString(1, username); ps.setString(2, username); ps.setString(3, username); @@ -349,6 +349,22 @@ public Set getPrivilegesTableForCurrentUser(final JdbcDatabase .collect(toSet()); } + static String getUsername(final JsonNode databaseConfig) { + final String host = databaseConfig.get("host").asText(); + final String username = databaseConfig.get("username").asText(); + + // Azure Postgres server has this username pattern: @. + // Inside Postgres, the true username is just . + if (username.contains("@") && host.endsWith("azure.com")) { + final String[] tokens = username.split("@"); + final String postgresUsername = tokens[0]; + LOGGER.info("Azure username \"{}\" is detected; use \"{}\" to check permission", username, postgresUsername); + return postgresUsername; + } + + return username; + } + @Override protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set internalSchemas) { return false; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 8cf536879eda..3ded33399b4c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -34,6 +34,7 @@ import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -359,4 +360,17 @@ void testIsCdc() { assertTrue(PostgresSource.isCdc(config)); } + @Test + void testGetUsername() { + final String username = "airbyte-user"; + + // normal host + final JsonNode normalConfig = Jsons.jsonNode(Map.of("username", username, "host", "airbyte.aws.com")); + assertEquals(username, PostgresSource.getUsername(normalConfig)); + + // azure host + final JsonNode azureConfig = Jsons.jsonNode(Map.of("username", username + "@airbyte", "host", "airbyte.azure.com")); + assertEquals(username, PostgresSource.getUsername(azureConfig)); + } + } From 80714f2964385a7208c5ef118312decd8fec19d9 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 12 May 2022 23:00:32 -0700 Subject: [PATCH 2/5] Bump version --- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- docs/integrations/sources/postgres.md | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index b72ad562fbb9..a0250be06dc3 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.14 +LABEL io.airbyte.version=0.4.15 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index cc4f371737a2..b527cc286d22 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.14 +LABEL io.airbyte.version=0.4.15 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index d632904fdbaa..ae92f27ab7d8 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -270,6 +270,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.15 | 2022-05-13 | [12834](https://github.com/airbytehq/airbyte/pull/12834) | Fix the bug that the connector returns empty catalog for Azure Postgres database | | 0.4.14 | 2022-05-08 | [12689](https://github.com/airbytehq/airbyte/pull/12689) | Add table retrieval according to role-based `SELECT` privilege | | 0.4.13 | 2022-05-05 | [10230](https://github.com/airbytehq/airbyte/pull/10230) | Explicitly set null value for field in json | | 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | @@ -278,7 +279,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | | 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | | 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | From 90d9c0759a42e30149a16fe0a97d33ecbcfea678 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 12 May 2022 23:10:52 -0700 Subject: [PATCH 3/5] Fix test --- .../integrations/source/postgres/PostgresSource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 01dfa386c703..bea9557ec1ae 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -349,8 +349,12 @@ public Set getPrivilegesTableForCurrentUser(final JdbcDatabase .collect(toSet()); } + @VisibleForTesting static String getUsername(final JsonNode databaseConfig) { - final String host = databaseConfig.get("host").asText(); + final String host = databaseConfig.get("host") == null + // host can be null in test + ? "" + : databaseConfig.get("host").asText(); final String username = databaseConfig.get("username").asText(); // Azure Postgres server has this username pattern: @. From 53e8262c5a3e6247f10455891f99b0e37cd38761 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 12 May 2022 23:32:04 -0700 Subject: [PATCH 4/5] Replace host with jdbc_url --- .../integrations/source/postgres/PostgresSource.java | 8 +++----- .../integrations/source/postgres/PostgresSourceTest.java | 8 ++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index bea9557ec1ae..bba6515a244c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -351,15 +351,13 @@ public Set getPrivilegesTableForCurrentUser(final JdbcDatabase @VisibleForTesting static String getUsername(final JsonNode databaseConfig) { - final String host = databaseConfig.get("host") == null - // host can be null in test - ? "" - : databaseConfig.get("host").asText(); + final String jdbcUrl = databaseConfig.get("jdbc_url").asText(); final String username = databaseConfig.get("username").asText(); // Azure Postgres server has this username pattern: @. // Inside Postgres, the true username is just . - if (username.contains("@") && host.endsWith("azure.com")) { + // The jdbc_url is constructed in the toDatabaseConfigStatic method. + if (username.contains("@") && jdbcUrl.contains("azure.com:")) { final String[] tokens = username.split("@"); final String postgresUsername = tokens[0]; LOGGER.info("Azure username \"{}\" is detected; use \"{}\" to check permission", username, postgresUsername); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 3ded33399b4c..c8cd44c8adba 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -365,11 +365,15 @@ void testGetUsername() { final String username = "airbyte-user"; // normal host - final JsonNode normalConfig = Jsons.jsonNode(Map.of("username", username, "host", "airbyte.aws.com")); + final JsonNode normalConfig = Jsons.jsonNode(Map.of( + "username", username, + "jdbc_url", "jdbc:postgresql://airbyte.database.com:5432:airbyte")); assertEquals(username, PostgresSource.getUsername(normalConfig)); // azure host - final JsonNode azureConfig = Jsons.jsonNode(Map.of("username", username + "@airbyte", "host", "airbyte.azure.com")); + final JsonNode azureConfig = Jsons.jsonNode(Map.of( + "username", username + "@airbyte", + "jdbc_url", "jdbc:postgresql://airbyte.azure.com:5432:airbyte")); assertEquals(username, PostgresSource.getUsername(azureConfig)); } From 977f6f108f79e8030a2b37ec2fe028dfe1ca78fe Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 13 May 2022 08:20:55 +0000 Subject: [PATCH 5/5] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 23876bf2f510..e77998c6131a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -715,7 +715,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.14 + dockerImageTag: 0.4.15 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4f16ad96510b..4d0669ed6655 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6540,7 +6540,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.14" +- dockerImage: "airbyte/source-postgres:0.4.15" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: