From d46058e456668012d334015674554a815572899c Mon Sep 17 00:00:00 2001 From: noahkawasaki-airbyte <103465980+noahkawasaki-airbyte@users.noreply.github.com> Date: Mon, 25 Apr 2022 08:12:31 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Destination=20Postgres:=20Add=20?= =?UTF-8?q?jdbc=5Furl=5Fparams=20input=20(#12195)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add jdbc_url_params input for Destination Postgres --- .../destination-postgres/Dockerfile | 2 +- .../postgres/PostgresDestination.java | 26 +++++--- .../src/main/resources/spec.json | 6 ++ .../postgres/PostgresDestinationTest.java | 59 +++++++++++++++---- docs/integrations/destinations/postgres.md | 7 ++- 5 files changed, 78 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 28fdf5c26b1b..5f354f799f1d 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.18 +LABEL io.airbyte.version=0.3.19 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 73e4c81be33a..5d1daf58e728 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -25,6 +25,13 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest public static final String DRIVER_CLASS = "org.postgresql.Driver"; public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + public static final String DATABASE_KEY = "database"; + public static final String JDBC_URL_KEY = "jdbc_url"; + public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; + public static final String PASSWORD_KEY = "password"; + public static final String SSL_KEY = "ssl"; + public static final String USERNAME_KEY = "username"; + public static final String SCHEMA_KEY = "schema"; static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", @@ -55,21 +62,26 @@ public JsonNode toJdbcConfig(final JsonNode config) { final String jdbcUrl = String.format("jdbc:postgresql://%s:%s/%s?", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText()); + config.get(DATABASE_KEY).asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl) - .put("schema", schema); + .put(USERNAME_KEY, config.get(USERNAME_KEY).asText()) + .put(JDBC_URL_KEY, jdbcUrl) + .put(SCHEMA_KEY, schema); - if (config.has("password")) { - configBuilder.put("password", config.get("password").asText()); + if (config.has(PASSWORD_KEY)) { + configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText()); } + + if (config.has(JDBC_URL_PARAMS_KEY)) { + configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY).asText()); + } + return Jsons.jsonNode(configBuilder.build()); } private boolean useSsl(final JsonNode config) { - return !config.has("ssl") || config.get("ssl").asBoolean(); + return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json index bfb18a87f2fb..1d475c9cf8b0 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json @@ -60,6 +60,12 @@ "type": "boolean", "default": false, "order": 6 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 7 } } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index f66387241dca..db9082f1371c 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -50,16 +49,33 @@ public class PostgresDestinationTest { private JsonNode config; - private static final Map CONFIG_WITH_SSL = ImmutableMap.of( - "host", "localhost", - "port", "1337", - "username", "user", - "database", "db"); + private static final String EXPECTED_JDBC_URL = "jdbc:postgresql://localhost:1337/db?"; - private static final Map CONFIG_NO_SSL = MoreMaps.merge( - CONFIG_WITH_SSL, - ImmutableMap.of( - "ssl", "false")); + private JsonNode buildConfigNoJdbcParameters() { + return Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db")); + } + + private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { + return Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db", + "jdbc_url_params", extraParam)); + } + + private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { + return Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db", + "ssl", false)); + } @BeforeAll static void init() { @@ -77,17 +93,36 @@ static void cleanUp() { PSQL_DB.close(); } + @Test + void testJdbcUrlAndConfigNoExtraParams() { + final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigNoJdbcParameters()); + assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText()); + } + + @Test + void testJdbcUrlEmptyExtraParams() { + final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText()); + } + + @Test + void testJdbcUrlExtraParams() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText()); + } + @Test void testDefaultParamsNoSSL() { final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_NO_SSL)); + buildConfigNoExtraJdbcParametersWithoutSsl()); assertEquals(new HashMap<>(), defaultProperties); } @Test void testDefaultParamsWithSSL() { final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_WITH_SSL)); + buildConfigNoJdbcParameters()); assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties); } diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index f66af37a34a1..be0f79a2db0f 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -63,6 +63,8 @@ You should now have all the requirements needed to configure Postgres as a desti * **Password** * **Default Schema Name** * **Database** +* **JDBC URL Params** (optional) + ## Naming Conventions @@ -84,8 +86,9 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------| -| 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | +| 0.3.19 | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input | +| 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | | 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key |