Skip to content

Commit

Permalink
[14003] source-oracle: added custom jdbc field (#14092)
Browse files Browse the repository at this point in the history
* [14003] source-oracle: added custom jdbc field
  • Loading branch information
etsybaev authored Jun 24, 2022
1 parent 11ea066 commit cc2b82c
Show file tree
Hide file tree
Showing 9 changed files with 164 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@
- name: Oracle DB
sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
dockerRepository: airbyte/source-oracle
dockerImageTag: 0.3.15
dockerImageTag: 0.3.17
documentationUrl: https://docs.airbyte.io/integrations/sources/oracle
icon: oracle.svg
sourceType: database
Expand Down
17 changes: 15 additions & 2 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6066,7 +6066,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-oracle:0.3.15"
- dockerImage: "airbyte/source-oracle:0.3.17"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle"
connectionSpecification:
Expand All @@ -6084,6 +6084,7 @@
title: "Host"
description: "Hostname of the database."
type: "string"
order: 1
port:
title: "Port"
description: "Port of the database.\nOracle Corporations recommends the\
Expand All @@ -6094,18 +6095,22 @@
minimum: 0
maximum: 65536
default: 1521
order: 2
sid:
title: "SID (Oracle System Identifier)"
type: "string"
order: 3
username:
title: "User"
description: "The username which is used to access the database."
type: "string"
order: 4
password:
title: "Password"
description: "The password associated with the username."
type: "string"
airbyte_secret: true
order: 5
schemas:
title: "Schemas"
description: "The list of schemas to sync from. Defaults to user. Case sensitive."
Expand All @@ -6114,12 +6119,20 @@
type: "string"
minItems: 1
uniqueItems: true
order: 6
jdbc_url_params:
title: "JDBC URL Params"
description: "Additional properties to pass to the JDBC URL string when\
\ connecting to the database formatted as 'key=value' pairs separated\
\ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
type: "string"
order: 7
encryption:
title: "Encryption"
type: "object"
description: "The encryption method with is used when communicating with\
\ the database."
order: 6
order: 8
oneOf:
- title: "Unencrypted"
additionalProperties: false
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-oracle/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ ENV TZ UTC
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1

LABEL io.airbyte.version=0.3.15
LABEL io.airbyte.version=0.3.17
LABEL io.airbyte.name=airbyte/source-oracle
1 change: 1 addition & 0 deletions airbyte-integrations/connectors/source-oracle/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ dependencies {
implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0"

testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
testImplementation project(":airbyte-json-validation")
testImplementation project(':airbyte-test-utils')

testImplementation 'org.apache.commons:commons-lang3:3.11'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ public JsonNode toDatabaseConfig(final JsonNode config) {
schemas.add(schema.asText());
}
}

if (config.get("jdbc_url_params") != null && !config.get("jdbc_url_params").asText().isEmpty()) {
additionalParameters.addAll(List.of(config.get("jdbc_url_params").asText().split("&")));
}

if (!additionalParameters.isEmpty()) {
final String connectionParams = String.join(getJdbcParameterDelimiter(), additionalParameters);
configBuilder.put("connection_properties", connectionParams);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,30 +10,35 @@
"host": {
"title": "Host",
"description": "Hostname of the database.",
"type": "string"
"type": "string",
"order": 1
},
"port": {
"title": "Port",
"description": "Port of the database.\nOracle Corporations recommends the following port numbers:\n1521 - Default listening port for client connections to the listener. \n2484 - Recommended and officially registered listening port for client connections to the listener using TCP/IP with SSL",
"type": "integer",
"minimum": 0,
"maximum": 65536,
"default": 1521
"default": 1521,
"order": 2
},
"sid": {
"title": "SID (Oracle System Identifier)",
"type": "string"
"type": "string",
"order": 3
},
"username": {
"title": "User",
"description": "The username which is used to access the database.",
"type": "string"
"type": "string",
"order": 4
},
"password": {
"title": "Password",
"description": "The password associated with the username.",
"type": "string",
"airbyte_secret": true
"airbyte_secret": true,
"order": 5
},
"schemas": {
"title": "Schemas",
Expand All @@ -43,13 +48,20 @@
"type": "string"
},
"minItems": 1,
"uniqueItems": true
"uniqueItems": true,
"order": 6
},
"jdbc_url_params": {
"title": "JDBC URL Params",
"description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
"type": "string",
"order": 7
},
"encryption": {
"title": "Encryption",
"type": "object",
"description": "The encryption method with is used when communicating with the database.",
"order": 6,
"order": 8,
"oneOf": [
{
"title": "Unencrypted",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package io.airbyte.integrations.source.oracle;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import com.fasterxml.jackson.databind.JsonNode;
Expand All @@ -23,6 +24,7 @@
import io.airbyte.protocol.models.AirbyteMessage.Type;
import io.airbyte.protocol.models.AirbyteRecordMessage;
import io.airbyte.protocol.models.AirbyteStateMessage;
import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType;
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
import io.airbyte.protocol.models.ConnectorSpecification;
import io.airbyte.protocol.models.DestinationSyncMode;
Expand Down Expand Up @@ -227,6 +229,7 @@ void testReadOneTableIncrementallyTwice() throws Exception {
expectedMessages.add(new AirbyteMessage()
.withType(Type.STATE)
.withState(new AirbyteStateMessage()
.withType(AirbyteStateType.LEGACY)
.withData(Jsons.jsonNode(new DbState()
.withCdc(false)
.withStreams(Lists.newArrayList(new DbStreamState()
Expand All @@ -237,7 +240,7 @@ void testReadOneTableIncrementallyTwice() throws Exception {

setEmittedAtToNull(actualMessagesSecondSync);

assertTrue(expectedMessages.size() == actualMessagesSecondSync.size());
assertArrayEquals(expectedMessages.toArray(), actualMessagesSecondSync.toArray());
assertTrue(expectedMessages.containsAll(actualMessagesSecondSync));
assertTrue(actualMessagesSecondSync.containsAll(expectedMessages));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.source.oracle;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.resources.MoreResources;
import io.airbyte.protocol.models.ConnectorSpecification;
import io.airbyte.validation.json.JsonSchemaValidator;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

/**
* Tests that the Oracle spec passes JsonSchema validation. While this may seem like overkill, we
* are doing it because there are some gotchas in correctly configuring the oneOf.
*/
public class OracleSpecTest {

private static final String CONFIGURATION = """
{
"host": "localhost",
"port": 1521,
"sid": "ora_db",
"username": "ora",
"password": "pwd",
"schemas": [
"public"
],
"jdbc_url_params": "property1=pValue1&property2=pValue2"
}
""";

private static JsonNode schema;
private static JsonSchemaValidator validator;

@BeforeAll
static void init() throws IOException {
final String spec = MoreResources.readResource("spec.json");
final File schemaFile = IOs.writeFile(Files.createTempDirectory(Path.of("/tmp"), "pg-spec-test"), "schema.json", spec).toFile();
schema = JsonSchemaValidator.getSchema(schemaFile).get("connectionSpecification");
validator = new JsonSchemaValidator();
}

@Test
void testHostMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("host");
assertFalse(validator.test(schema, config));
}

@Test
void testPortMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("port");
assertFalse(validator.test(schema, config));
}

@Test
void testSsidMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("sid");
assertFalse(validator.test(schema, config));
}

@Test
void testUsernameMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("username");
assertFalse(validator.test(schema, config));
}

@Test
void testPasswordMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("password");
assertTrue(validator.test(schema, config));
}

@Test
void testSchemaMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("schemas");
assertTrue(validator.test(schema, config));
}

@Test
void testAdditionalJdbcParamMissing() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
((ObjectNode) config).remove("jdbc_url_params");
assertTrue(validator.test(schema, config));
}

@Test
void testWithJdbcAdditionalProperty() {
final JsonNode config = Jsons.deserialize(CONFIGURATION);
assertTrue(validator.test(schema, config));
}

@Test
void testJdbcAdditionalProperty() throws Exception {
final ConnectorSpecification spec = new OracleSource().spec();
assertNotNull(spec.getConnectionSpecification().get("properties").get("jdbc_url_params"));
}

}
1 change: 1 addition & 0 deletions docs/integrations/sources/oracle.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv

| Version | Date | Pull Request | Subject |
|:--------| :--- | :--- |:------------------------------------------------|
| 0.3.17 | 2022-06-24 | [14092](https://github.com/airbytehq/airbyte/pull/14092) | Introduced a custom jdbc param field |
| 0.3.16 | 2022-06-22 | [13997](https://github.com/airbytehq/airbyte/pull/13997) | Fixed tests |
| 0.3.15 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |
| 0.3.14 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats |
Expand Down

0 comments on commit cc2b82c

Please sign in to comment.