diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 4875cef67707..6f79ae9d1f46 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -132,8 +132,6 @@ jobs: POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} - ORACLE_SSH_KEY_TEST_CREDS: ${{ secrets.ORACLE_SSH_KEY_TEST_CREDS }} - ORACLE_SSH_PWD_TEST_CREDS: ${{ secrets.ORACLE_SSH_PWD_TEST_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 55216adaa42e..0275fe11d14f 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -128,8 +128,6 @@ jobs: POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} - ORACLE_SSH_KEY_TEST_CREDS: ${{ secrets.ORACLE_SSH_KEY_TEST_CREDS }} - ORACLE_SSH_PWD_TEST_CREDS: ${{ secrets.ORACLE_SSH_PWD_TEST_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json index f989cea25365..ec47ebbd5d9f 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", "name": "Oracle (Alpha)", "dockerRepository": "airbyte/destination-oracle", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e8d413893af7..533b3d7f8603 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -73,7 +73,7 @@ - destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72 name: Oracle (Alpha) dockerRepository: airbyte/destination-oracle - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle - destinationDefinitionId: 9f760101-60ae-462f-9ee6-b7a9dafd454d name: Kafka diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 4a42b71c7c3f..aa450871760d 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index 611e1e26a211..2532c984ec7d 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.testcontainers:oracle-xe:1.15.2' integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index da822bbb5662..9002ec51f505 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -30,13 +30,17 @@ import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class OracleDestination extends AbstractJdbcDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(OracleDestination.class); + public static final List HOST_KEY = List.of("host"); + public static final List PORT_KEY = List.of("port"); public static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver"; @@ -66,7 +70,7 @@ public JsonNode toJdbcConfig(JsonNode config) { } public static void main(String[] args) throws Exception { - final Destination destination = new OracleDestination(); + final Destination destination = new SshWrappedDestination(new OracleDestination(), HOST_KEY, PORT_KEY); LOGGER.info("starting destination: {}", OracleDestination.class); new IntegrationRunner(destination).run(args); LOGGER.info("completed destination: {}", OracleDestination.class); diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java index 445475a38194..d506fe78c244 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java @@ -59,7 +59,9 @@ public String convertStreamName(String input) { if (!result.isEmpty() && result.charAt(0) == '_') { result = result.substring(1); } - return maxStringLength(result, 128); + // prior to Oracle version 12.2, identifiers are not allowed to exceed 30 characters in length. + // However, from version 12.2 they can be up to 128 bytes long. (Note: bytes, not characters). + return maxStringLength(result, 30); } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index ba1e9435c749..1470018a11f8 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -56,7 +56,10 @@ public void createSchemaIfNotExists(JdbcDatabase database, String schemaName) th LOGGER.warn("Schema " + schemaName + " is not found! Trying to create a new one."); final String query = String.format("create user %s identified by %s quota unlimited on %s", schemaName, schemaName, tablespace); + // need to grant privileges to new user / this option is not mandatory for Oracle DB 18c or higher + final String privileges = String.format("GRANT ALL PRIVILEGES TO %s", schemaName); database.execute(query); + database.execute(privileges); } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java new file mode 100644 index 000000000000..45e3abae163a --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java @@ -0,0 +1,36 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import io.airbyte.integrations.base.ssh.SshTunnel; + +public class SshKeyOracleDestinationAcceptanceTest extends SshOracleDestinationAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_KEY_AUTH; + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java new file mode 100644 index 000000000000..fd80ee8f4d64 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -0,0 +1,199 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.functional.CheckedFunction; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.jooq.JSONFormat; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.OracleContainer; + +public abstract class SshOracleDestinationAcceptanceTest extends DestinationAcceptanceTest { + + private static final JSONFormat JSON_FORMAT = new JSONFormat().recordFormat(JSONFormat.RecordFormat.OBJECT); + + private final ExtendedNameTransformer namingResolver = new OracleNameTransformer(); + + private final String schemaName = "TEST_ORCL"; + + private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); + + private OracleContainer db; + + public abstract SshTunnel.TunnelMethod getTunnelMethod(); + + @Override + protected String getImageName() { + return "airbyte/destination-oracle:dev"; + } + + @Override + protected JsonNode getConfig() throws IOException, InterruptedException { + return sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db).put("schema", schemaName)); + } + + public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { + return ImmutableMap.builder() + .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() + .getNetworks() + .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) + .getIpAddress())) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("port", db.getExposedPorts().get(0)) + .put("sid", db.getSid()) + .put("schemas", List.of("JDBC_SPACE")) + .put("ssl", false); + } + + @Override + protected List resolveIdentifier(String identifier) { + final List result = new ArrayList<>(); + final String resolved = namingResolver.getIdentifier(identifier); + result.add(identifier); + result.add(resolved); + if (!resolved.startsWith("\"")) { + result.add(resolved.toLowerCase()); + result.add(resolved.toUpperCase()); + } + return result; + } + + @Override + protected JsonNode getFailCheckConfig() throws Exception { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("password", "wrong password"); + return clone; + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, String streamName, String namespace, JsonNode streamSchema) throws Exception { + List jsonNodes = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); + return jsonNodes + .stream() + .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase()).asText())) + .collect(Collectors.toList()); + } + + @Override + protected List retrieveNormalizedRecords(TestDestinationEnv env, String streamName, String namespace) + throws Exception { + String tableName = namingResolver.getIdentifier(streamName); + return retrieveRecordsFromTable(tableName, namespace); + } + + private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { + final JsonNode config = getConfig(); + return SshTunnel.sshWrap( + config, + OracleDestination.HOST_KEY, + OracleDestination.PORT_KEY, + (CheckedFunction, Exception>) mangledConfig -> getDatabaseFromConfig(mangledConfig) + .query( + ctx -> ctx + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT))) + .stream() + .map(r -> r.formatJSON(JSON_FORMAT)) + .map(Jsons::deserialize) + .collect(Collectors.toList())); + } + + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { + startTestContainers(); + SshTunnel.sshWrap( + getConfig(), + OracleDestination.HOST_KEY, + OracleDestination.PORT_KEY, + mangledConfig -> { + Database databaseFromConfig = getDatabaseFromConfig(mangledConfig); + databaseFromConfig.query(ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); + databaseFromConfig.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); + }); + } + + private void startTestContainers() { + sshBastionContainer.initAndStartBastion(); + initAndStartJdbcContainer(); + } + + private void initAndStartJdbcContainer() { + db = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withNetwork(sshBastionContainer.getNetWork()); + db.start(); + } + + private Database getDatabaseFromConfig(final JsonNode config) { + return Databases.createDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + null); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + SshTunnel.sshWrap( + getConfig(), + OracleDestination.HOST_KEY, + OracleDestination.PORT_KEY, + mangledConfig -> { + Database databaseFromConfig = getDatabaseFromConfig(mangledConfig); + databaseFromConfig.query(ctx -> ctx.fetch(String.format("DROP USER %s CASCADE", schemaName))); + }); + + sshBastionContainer.stopAndCloseContainers(db); + } + + @Override + protected boolean supportsDBT() { + return true; + } + + @Override + protected boolean implementsNamespaces() { + return true; + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java new file mode 100644 index 000000000000..47d5885cda85 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java @@ -0,0 +1,36 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import io.airbyte.integrations.base.ssh.SshTunnel; + +public class SshPasswordOracleDestinationAcceptanceTest extends SshOracleDestinationAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; + } + +} diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index 7b99fcbb511d..c4066a2adaee 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -25,41 +25,82 @@ package io.airbyte.integrations.source.oracle; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.CatalogHelpers; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaPrimitive; -import io.airbyte.protocol.models.SyncMode; -import java.nio.file.Path; +import io.airbyte.protocol.models.*; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Objects; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.OracleContainer; public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; + private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); + private OracleContainer db; private JsonNode config; - public abstract Path getConfigFilePath(); + public abstract SshTunnel.TunnelMethod getTunnelMethod(); @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - config = Jsons.deserialize(IOs.readFile(getConfigFilePath())); + startTestContainers(); + config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); + populateDatabaseTestData(); } + private void populateDatabaseTestData() throws Exception { + JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver"); + + database.execute(connection -> { + connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); + connection.createStatement().execute("CREATE TABLE jdbc_space.id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power BINARY_DOUBLE)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (1,'goku', BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (NULL, 'piccolo', -BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("CREATE TABLE jdbc_space.starships(id INTEGER, name VARCHAR(200))"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (1,'enterprise-d')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (2, 'defiant')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); + }); + + database.close(); + } + + ; + @Override protected void tearDown(final TestDestinationEnv testEnv) { + sshBastionContainer.stopAndCloseContainers(db); + } + private void startTestContainers() { + sshBastionContainer.initAndStartBastion(); + initAndStartJdbcContainer(); + } + + private void initAndStartJdbcContainer() { + db = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withNetwork(sshBastionContainer.getNetWork()); + db.start(); } @Override @@ -72,6 +113,19 @@ protected ConnectorSpecification getSpec() throws Exception { return SshHelpers.getSpecAndInjectSsh(); } + public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { + return ImmutableMap.builder() + .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() + .getNetworks() + .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) + .getIpAddress())) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("port", db.getExposedPorts().get(0)) + .put("sid", db.getSid()) + .put("schemas", List.of("JDBC_SPACE")); + } + @Override protected JsonNode getConfig() { return config; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java index 450a18cdb532..73604cd47486 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java @@ -24,13 +24,13 @@ package io.airbyte.integrations.source.oracle; -import java.nio.file.Path; +import io.airbyte.integrations.base.ssh.SshTunnel; public class SshKeyOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override - public Path getConfigFilePath() { - return Path.of("secrets/ssh-key-config.json"); + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_KEY_AUTH; } } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java index f78b3241133a..4fb05a21e846 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java @@ -24,13 +24,13 @@ package io.airbyte.integrations.source.oracle; -import java.nio.file.Path; +import io.airbyte.integrations.base.ssh.SshTunnel; public class SshPasswordOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override - public Path getConfigFilePath() { - return Path.of("secrets/ssh-pwd-config.json"); + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; } } diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index d1bd9393a9ae..e26d28847f64 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -62,10 +62,28 @@ You should now have all the requirements needed to configure Oracle as a destina * **Username** * **Password** * **Database** +* +## Connection to Oracle via an SSH Tunnel + +Airbyte has the ability to connect to a Oracle instance via an SSH Tunnel. The reason you might want to do this because it is not possible (or against security policy) to connect to the database directly (e.g. it does not have a public IP address). + +When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (a.k.a. a bastion sever) that _does_ have direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. + +Using this feature requires additional configuration, when creating the source. We will talk through what each piece of configuration means. +1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. +2. `SSH Tunnel Method` defaults to `No Tunnel` (meaning a direct connection). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. + 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel (see below for more information on generating this key). + 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. +3. `SSH Tunnel Jump Server Host` refers to the intermediate (bastion) server that Airbyte will connect to. This should be a hostname or an IP Address. +4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default. +5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the Oracle username. +6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. +7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2021-09-28 | [#6370](https://github.com/airbytehq/airbyte/pull/6370)| Add SSH Support for Oracle Destination | | 0.1.7 | 2021-08-30 | [#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | | 0.1.6 | 2021-08-23 | [#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | | 0.1.5 | 2021-08-10 | [#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 324271012f20..703cb1683120 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -102,8 +102,6 @@ write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS" write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" write_standard_creds source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" write_standard_creds source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" -write_standard_creds source-oracle "$ORACLE_SSH_KEY_TEST_CREDS" "ssh-key-config.json" -write_standard_creds source-oracle "$ORACLE_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" write_standard_creds source-posthog "$POSTHOG_TEST_CREDS" write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS"