Skip to content

Commit

Permalink
Source DB2: Add custom jdbc params (#16354)
Browse files Browse the repository at this point in the history
* DB2 add custom jdbc params

* bump version

* auto-bump connector version [ci skip]

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
  • Loading branch information
yurii-bidiuk and octavia-squidington-iii authored Sep 6, 2022
1 parent ce201bc commit dd6baf9
Show file tree
Hide file tree
Showing 10 changed files with 125 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@
- name: IBM Db2
sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
dockerRepository: airbyte/source-db2
dockerImageTag: 0.1.15
dockerImageTag: 0.1.16
documentationUrl: https://docs.airbyte.io/integrations/sources/db2
icon: db2.svg
sourceType: database
Expand Down
11 changes: 9 additions & 2 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4016,7 +4016,7 @@
- - "client_secret"
oauthFlowOutputParameters:
- - "refresh_token"
- dockerImage: "airbyte/source-db2:0.1.15"
- dockerImage: "airbyte/source-db2:0.1.16"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/db2"
connectionSpecification:
Expand Down Expand Up @@ -4059,11 +4059,18 @@
type: "string"
airbyte_secret: true
order: 4
jdbc_url_params:
description: "Additional properties to pass to the JDBC URL string when\
\ connecting to the database formatted as 'key=value' pairs separated\
\ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
title: "JDBC URL Params"
type: "string"
order: 5
encryption:
title: "Encryption"
type: "object"
description: "Encryption method to use when communicating with the database"
order: 5
order: 6
oneOf:
- title: "Unencrypted"
description: "Data transfer will not be encrypted."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.15
LABEL io.airbyte.version=0.1.16
LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,17 @@
"airbyte_secret": true,
"order": 4
},
"jdbc_url_params": {
"description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
"title": "JDBC URL Params",
"type": "string",
"order": 5
},
"encryption": {
"title": "Encryption",
"type": "object",
"description": "Encryption method to use when communicating with the database",
"order": 5,
"order": 6,
"oneOf": [
{
"title": "TLS Encrypted (verify certificate)",
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-db2/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-db2

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.15
LABEL io.airbyte.version=0.1.16
LABEL io.airbyte.name=airbyte/source-db2
1 change: 1 addition & 0 deletions airbyte-integrations/connectors/source-db2/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ dependencies {
testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
testImplementation project(':airbyte-test-utils')
testImplementation libs.connectors.testcontainers.db2
testImplementation project(":airbyte-json-validation")

integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package io.airbyte.integrations.source.db2;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.commons.functional.CheckedFunction;
import io.airbyte.commons.json.Jsons;
Expand Down Expand Up @@ -79,6 +80,10 @@ public JsonNode toDatabaseConfig(final JsonNode config) {
.build());
}

if (config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty()) {
((ObjectNode) result).put(JdbcUtils.JDBC_URL_PARAMS_KEY, config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText());
}

return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,17 @@
"airbyte_secret": true,
"order": 4
},
"jdbc_url_params": {
"description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
"title": "JDBC URL Params",
"type": "string",
"order": 5
},
"encryption": {
"title": "Encryption",
"type": "object",
"description": "Encryption method to use when communicating with the database",
"order": 5,
"order": 6,
"oneOf": [
{
"title": "Unencrypted",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.source.db2;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.resources.MoreResources;
import io.airbyte.protocol.models.ConnectorSpecification;
import io.airbyte.validation.json.JsonSchemaValidator;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

public class Db2SpecTest {

private static JsonNode schema;
private static JsonNode config;
private static String configText;
private static JsonSchemaValidator validator;

@BeforeAll
static void init() throws IOException {
configText = """
{
"host": "localhost",
"port": 1521,
"db": "db",
"username": "test",
"password": "password",
"jdbc_url_params": "property1=pValue1&property2=pValue2"
}
""";
final String spec = MoreResources.readResource("spec.json");
final File schemaFile = IOs.writeFile(Files.createTempDirectory(Path.of("/tmp"), "pg-spec-test"), "schema.json", spec).toFile();
schema = JsonSchemaValidator.getSchema(schemaFile).get("connectionSpecification");
validator = new JsonSchemaValidator();
}

@BeforeEach
void beforeEach() {
config = Jsons.deserialize(configText);
}

@Test
void testHostMissing() {
((ObjectNode) config).remove("host");
assertFalse(validator.test(schema, config));
}

@Test
void testPortMissing() {
((ObjectNode) config).remove("port");
assertFalse(validator.test(schema, config));
}

@Test
void testDatabaseMissing() {
((ObjectNode) config).remove("db");
assertFalse(validator.test(schema, config));
}

@Test
void testUsernameMissing() {
((ObjectNode) config).remove("username");
assertFalse(validator.test(schema, config));
}

@Test
void testPasswordMissing() {
((ObjectNode) config).remove("password");
assertFalse(validator.test(schema, config));
}

@Test
void testJdbcAdditionalProperty() throws Exception {
final ConnectorSpecification spec = new Db2Source().spec();
assertNotNull(spec.getConnectionSpecification().get("properties").get("jdbc_url_params"));
}

}
3 changes: 2 additions & 1 deletion docs/integrations/sources/db2.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ You can also enter your own password for the keystore, but if you don't, the pas

| Version | Date | Pull Request | Subject |
| :--- | :--- | :--- | :--- |
| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params |
| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
Expand Down

0 comments on commit dd6baf9

Please sign in to comment.