Skip to content

Commit

Permalink
🎉 Destination Postgres: Add jdbc_url_params input (#12195)
Browse files Browse the repository at this point in the history 
* Add jdbc_url_params input for Destination Postgres
  • Loading branch information
@noahkawasaki-airbyte
noahkawasaki-airbyte authored Apr 25, 2022
1 parent c42b00e commit d46058e
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 22 deletions.
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-postgres/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.3.18
LABEL io.airbyte.version=0.3.19
LABEL io.airbyte.name=airbyte/destination-postgres
26 changes: 19 additions & 7 deletions ...tgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest
public static final String DRIVER_CLASS = "org.postgresql.Driver";
public static final List<String> HOST_KEY = List.of("host");
public static final List<String> PORT_KEY = List.of("port");
public static final String DATABASE_KEY = "database";
public static final String JDBC_URL_KEY = "jdbc_url";
public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
public static final String PASSWORD_KEY = "password";
public static final String SSL_KEY = "ssl";
public static final String USERNAME_KEY = "username";
public static final String SCHEMA_KEY = "schema";

static final Map<String, String> SSL_JDBC_PARAMETERS = ImmutableMap.of(
"ssl", "true",
Expand Down Expand Up @@ -55,21 +62,26 @@ public JsonNode toJdbcConfig(final JsonNode config) {
final String jdbcUrl = String.format("jdbc:postgresql://%s:%s/%s?",
config.get("host").asText(),
config.get("port").asText(),
config.get("database").asText());
config.get(DATABASE_KEY).asText());

final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
.put("username", config.get("username").asText())
.put("jdbc_url", jdbcUrl)
.put("schema", schema);
.put(USERNAME_KEY, config.get(USERNAME_KEY).asText())
.put(JDBC_URL_KEY, jdbcUrl)
.put(SCHEMA_KEY, schema);

if (config.has("password")) {
configBuilder.put("password", config.get("password").asText());
if (config.has(PASSWORD_KEY)) {
configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText());
}

if (config.has(JDBC_URL_PARAMS_KEY)) {
configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY).asText());
}

return Jsons.jsonNode(configBuilder.build());
}

private boolean useSsl(final JsonNode config) {
return !config.has("ssl") || config.get("ssl").asBoolean();
return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean();
}

public static void main(final String[] args) throws Exception {
Expand Down
6 changes: 6 additions & 0 deletions airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@
"type": "boolean",
"default": false,
"order": 6
},
"jdbc_url_params": {
"description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
"title": "JDBC URL Params",
"type": "string",
"order": 7
}
}
}
Expand Down
59 changes: 47 additions & 12 deletions ...s/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.map.MoreMaps;
import io.airbyte.db.jdbc.JdbcDatabase;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.base.AirbyteMessageConsumer;
Expand Down Expand Up @@ -50,16 +49,33 @@ public class PostgresDestinationTest {

private JsonNode config;

private static final Map<String, String> CONFIG_WITH_SSL = ImmutableMap.of(
"host", "localhost",
"port", "1337",
"username", "user",
"database", "db");
private static final String EXPECTED_JDBC_URL = "jdbc:postgresql://localhost:1337/db?";

private static final Map<String, String> CONFIG_NO_SSL = MoreMaps.merge(
CONFIG_WITH_SSL,
ImmutableMap.of(
"ssl", "false"));
private JsonNode buildConfigNoJdbcParameters() {
return Jsons.jsonNode(ImmutableMap.of(
"host", "localhost",
"port", 1337,
"username", "user",
"database", "db"));
}

private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) {
return Jsons.jsonNode(ImmutableMap.of(
"host", "localhost",
"port", 1337,
"username", "user",
"database", "db",
"jdbc_url_params", extraParam));
}

private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() {
return Jsons.jsonNode(ImmutableMap.of(
"host", "localhost",
"port", 1337,
"username", "user",
"database", "db",
"ssl", false));
}

@BeforeAll
static void init() {
Expand All @@ -77,17 +93,36 @@ static void cleanUp() {
PSQL_DB.close();
}

@Test
void testJdbcUrlAndConfigNoExtraParams() {
final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigNoJdbcParameters());
assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
}

@Test
void testJdbcUrlEmptyExtraParams() {
final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(""));
assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
}

@Test
void testJdbcUrlExtraParams() {
final String extraParam = "key1=value1&key2=value2&key3=value3";
final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam));
assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
}

@Test
void testDefaultParamsNoSSL() {
final Map<String, String> defaultProperties = new PostgresDestination().getDefaultConnectionProperties(
Jsons.jsonNode(CONFIG_NO_SSL));
buildConfigNoExtraJdbcParametersWithoutSsl());
assertEquals(new HashMap<>(), defaultProperties);
}

@Test
void testDefaultParamsWithSSL() {
final Map<String, String> defaultProperties = new PostgresDestination().getDefaultConnectionProperties(
Jsons.jsonNode(CONFIG_WITH_SSL));
buildConfigNoJdbcParameters());
assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties);
}

Expand Down
7 changes: 5 additions & 2 deletions docs/integrations/destinations/postgres.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ You should now have all the requirements needed to configure Postgres as a desti
* **Password**
* **Default Schema Name**
* **Database**
* **JDBC URL Params** (optional)


## Naming Conventions

Expand All @@ -84,8 +86,9 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the

| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------|
| 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 |
| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name |
| 0.3.19 | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input |
| 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 |
| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name |
| 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling |
| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option |
| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key |
Expand Down

0 comments on commit d46058e

Please sign in to comment.