From 68a8f78ee0c2488fad51edc48bd66e35ca7d0da8 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Fri, 15 Oct 2021 18:37:22 +0300 Subject: [PATCH 01/11] [Ticket 6435] Source-ClickHouse: added ssl support --- .../connectors/source-clickhouse/ReadMe.md | 5 + .../connectors/source-clickhouse/build.gradle | 2 +- .../source/clickhouse/ClickHouseSource.java | 19 +++- .../src/main/resources/spec.json | 6 ++ .../ClickHouseJdbcSourceAcceptanceTest.java | 3 +- .../sources/ClickHouseJdbcStressTest.java | 3 +- .../ClickHouseSourceAcceptanceTest.java | 3 +- ...SslClickHouseJdbcSourceAcceptanceTest.java | 98 +++++++++++++++++++ .../Clickhouse.Dockerfile | 9 ++ .../integrations-test-ssl/clickhouse_certs.sh | 15 +++ 10 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/source-clickhouse/ReadMe.md create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java create mode 100644 tools/integrations-test-ssl/Clickhouse.Dockerfile create mode 100644 tools/integrations-test-ssl/clickhouse_certs.sh diff --git a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md new file mode 100644 index 000000000000..10ce40eeeb75 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md @@ -0,0 +1,5 @@ +for ssl test custom image is used. To push it use +docker build -t airbyte/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile + + +under the tools\integration-tests-ssl dir \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index a32ff5bb98be..8cc18f1ac731 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -22,5 +22,5 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - integrationTestJavaImplementation "org.testcontainers:clickhouse:1.15.3" + integrationTestJavaImplementation "org.testcontainers:clickhouse:1.16.0" } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index b3a28517079b..7d3fea232919 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -35,6 +35,10 @@ public class ClickHouseSource extends AbstractJdbcSource implements Source { * https://clickhouse.tech/docs/en/operations/system-tables/columns/ to fetch the primary keys. */ + public static final List SSL_PARAMETERS = List.of( + "ssl=true", + "sslmode=none"); + @Override protected Map> discoverPrimaryKeys(JdbcDatabase database, List>> tableInfos) { @@ -73,13 +77,20 @@ public ClickHouseSource() { @Override public JsonNode toDatabaseConfig(JsonNode config) { + final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText())); + + // assume ssl if not explicitly mentioned. + if (!config.has("ssl") || config.get("ssl").asBoolean()) { + jdbcUrl.append("?").append(String.join("&", SSL_PARAMETERS)); + } + return Jsons.jsonNode(ImmutableMap.builder() .put("username", config.get("username").asText()) .put("password", config.get("password").asText()) - .put("jdbc_url", String.format("jdbc:clickhouse://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText())) + .put("jdbc_url", jdbcUrl.toString()) .build()); } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json index fa0c7a8c2e96..6d9250a966d8 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json @@ -32,6 +32,12 @@ "description": "Password associated with the username.", "type": "string", "airbyte_secret": true + }, + "ssl": { + "title": "SSL Connection", + "description": "Encrypt data using SSL.", + "type": "boolean", + "default": true } } } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index c7388a29eca7..0d6290e07f2d 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -76,7 +76,7 @@ public String primaryKeyClause(final List columns) { @Override @BeforeEach public void setup() throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.3.10.1-alpine"); + db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() @@ -85,6 +85,7 @@ public void setup() throws Exception { .put("database", SCHEMA_NAME) .put("username", db.getUsername()) .put("password", db.getPassword()) + .put("ssl", false) .build()); super.setup(); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java index 984e7b2ee5dd..c0a22f0ee81b 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java @@ -31,7 +31,7 @@ public Optional getDefaultSchemaName() { @Override @BeforeEach public void setup() throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.3.10.1-alpine"); + db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() @@ -40,6 +40,7 @@ public void setup() throws Exception { .put("database", SCHEMA_NAME) .put("username", db.getUsername()) .put("password", db.getPassword()) + .put("ssl", false) .build()); super.setup(); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index ca788401d9b8..ebeb9bff3814 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -86,7 +86,7 @@ protected List getRegexTests() { @Override protected void setupEnvironment(TestDestinationEnv environment) throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.3.10.1-alpine"); + db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() @@ -95,6 +95,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .put("database", SCHEMA_NAME) .put("username", db.getUsername()) .put("password", db.getPassword()) + .put("ssl", false) .build()); JdbcDatabase database = Databases.createJdbcDatabase( diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..4870b2f62840 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.clickhouse.ClickHouseSource; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.testcontainers.containers.GenericContainer; + +public class SslClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + + private static final String SCHEMA_NAME = "default"; + private GenericContainer db; + private JsonNode config; + + @Override + public boolean supportsSchemas() { + return false; + } + + @Override + public JsonNode getConfig() { + return Jsons.clone(config); + } + + @Override + public String getDriverClass() { + return ClickHouseSource.DRIVER_CLASS; + } + + @Override + public String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { + // ClickHouse requires Engine to be mentioned as part of create table query. + // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information + return String.format("CREATE TABLE %s(%s) %s", + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + + primaryKeyClause); + } + + @Override + @AfterEach + public void tearDown() throws SQLException { + db.close(); + db.stop(); + super.tearDown(); + } + + @Override + public String primaryKeyClause(final List columns) { + if (columns.isEmpty()) { + return ""; + } + + final StringBuilder clause = new StringBuilder(); + clause.append("("); + for (int i = 0; i < columns.size(); i++) { + clause.append(columns.get(i)); + if (i != (columns.size() - 1)) { + clause.append(","); + } + } + clause.append(")"); + return clause.toString(); + } + + @Override + @BeforeEach + public void setup() throws Exception { + db = new GenericContainer("airbyte/clickhouse-with-ssl:dev").withExposedPorts(8443); + db.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("port", db.getFirstMappedPort()) + .put("database", SCHEMA_NAME) + .put("username", "default") + .put("password", "") + .build()); + + super.setup(); + } + + @Override + public AbstractJdbcSource getJdbcSource() { + return new ClickHouseSource(); + } + +} diff --git a/tools/integrations-test-ssl/Clickhouse.Dockerfile b/tools/integrations-test-ssl/Clickhouse.Dockerfile new file mode 100644 index 000000000000..a4288be5af09 --- /dev/null +++ b/tools/integrations-test-ssl/Clickhouse.Dockerfile @@ -0,0 +1,9 @@ +FROM yandex/clickhouse-server:latest + +EXPOSE 8443 +EXPOSE 9000 +EXPOSE 8123 +EXPOSE 9009 + +COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ + diff --git a/tools/integrations-test-ssl/clickhouse_certs.sh b/tools/integrations-test-ssl/clickhouse_certs.sh new file mode 100644 index 000000000000..ab692baf9bcd --- /dev/null +++ b/tools/integrations-test-ssl/clickhouse_certs.sh @@ -0,0 +1,15 @@ +echo "Preparing certs" + +openssl req -subj "/CN=my.host.name" -new \ + -newkey rsa:2048 -days 365 -nodes -x509 \ + -keyout /etc/clickhouse-server/server.key \ + -out /etc/clickhouse-server/server.crt + +openssl dhparam -out /etc/clickhouse-server/dhparam.pem 1024 + +chown $(id -u clickhouse):$(id -g clickhouse) /etc/clickhouse-server/server.{key,crt} + +echo "8443" > /etc/clickhouse-server/config.d/https.xml + + +echo "Finished preparing certs" \ No newline at end of file From 80373ddd0f96c47ad117f28d16eab3f51a7590cc Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 18 Oct 2021 17:46:42 +0300 Subject: [PATCH 02/11] [Ticket 6716] Source-ClickHouse: added secure only connector --- .../.dockerignore | 3 + .../Dockerfile | 12 ++ .../ReadMe.md | 5 + .../acceptance-test-config.yml | 6 + .../build.gradle | 28 ++++ .../ClickHouseStrictEncryptSource.java | 40 ++++++ ...StrictEncryptJdbcSourceAcceptanceTest.java | 121 ++++++++++++++++++ .../resources/expected_spec.json | 38 ++++++ .../source/clickhouse/ClickHouseSource.java | 6 +- 9 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile new file mode 100644 index 000000000000..d5deee1c6c8b --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile @@ -0,0 +1,12 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-clickhouse-strict-sncrypt + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-clickhouse-strict-sncrypt diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md new file mode 100644 index 000000000000..10ce40eeeb75 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md @@ -0,0 +1,5 @@ +for ssl test custom image is used. To push it use +docker build -t airbyte/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile + + +under the tools\integration-tests-ssl dir \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml new file mode 100644 index 000000000000..e817af1e1929 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml @@ -0,0 +1,6 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-clickhouse-strict-sncrypt:dev +tests: + spec: + - spec_path: "src/test-integration/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle new file mode 100644 index 000000000000..2aab9fe0f5aa --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle @@ -0,0 +1,28 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] +} + +dependencies { + implementation project(':airbyte-db:lib') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:connectors:source-jdbc') + implementation project(':airbyte-integrations:connectors:source-relational-db') + implementation project(':airbyte-integrations:connectors:source-clickhouse') + implementation project(':airbyte-protocol:models') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1' + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse-strict-sncrypt') + integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + integrationTestJavaImplementation "org.testcontainers:clickhouse:1.16.0" +} diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java new file mode 100644 index 000000000000..5bc51a9bdf3a --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.clickhouse; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; +import io.airbyte.protocol.models.ConnectorSpecification; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ClickHouseStrictEncryptSource extends SpecModifyingSource implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(ClickHouseStrictEncryptSource.class); + + public ClickHouseStrictEncryptSource() { + super(ClickHouseSource.getWrappedSource()); + } + + @Override + public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { + final ConnectorSpecification spec = Jsons.clone(originalSpec); + // SSL property should be enabled by default for secure versions of connectors + // that can be used in the Airbyte cloud. User should not be able to change this property. + ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove("ssl"); + return spec; + } + + public static void main(String[] args) throws Exception { + final Source source = new ClickHouseStrictEncryptSource(); + LOGGER.info("starting source: {}", ClickHouseStrictEncryptSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", ClickHouseStrictEncryptSource.class); + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..c009358bd9da --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static org.junit.Assert.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.source.clickhouse.ClickHouseSource; +import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; + +public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + + private static final String SCHEMA_NAME = "default"; + private GenericContainer db; + private JsonNode config; + + @Override + public boolean supportsSchemas() { + return false; + } + + @Override + public JsonNode getConfig() { + return Jsons.clone(config); + } + + @Override + public String getDriverClass() { + return ClickHouseSource.DRIVER_CLASS; + } + + @Override + public String createTableQuery(final String tableName, + final String columnClause, + final String primaryKeyClause) { + // ClickHouse requires Engine to be mentioned as part of create table query. + // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information + return String.format("CREATE TABLE %s(%s) %s", + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + + primaryKeyClause); + } + + @Override + @AfterEach + public void tearDown() throws SQLException { + db.close(); + db.stop(); + super.tearDown(); + } + + @Override + public String primaryKeyClause(final List columns) { + if (columns.isEmpty()) { + return ""; + } + + final StringBuilder clause = new StringBuilder(); + clause.append("("); + for (int i = 0; i < columns.size(); i++) { + clause.append(columns.get(i)); + if (i != (columns.size() - 1)) { + clause.append(","); + } + } + clause.append(")"); + return clause.toString(); + } + + @Override + @BeforeEach + public void setup() throws Exception { + db = new GenericContainer("airbyte/clickhouse-with-ssl:dev").withExposedPorts(8443); + db.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("port", db.getFirstMappedPort()) + .put("database", SCHEMA_NAME) + .put("username", "default") + .put("password", "") + .build()); + + super.setup(); + } + + @Override + public AbstractJdbcSource getJdbcSource() { + return new ClickHouseSource(); + } + + @Override + public Source getSource() { + return new ClickHouseStrictEncryptSource(); + } + + @Test + void testSpec() throws Exception { + final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification expected = + Jsons.deserialize(MoreResources.readResource("expected_spec.json"), + ConnectorSpecification.class); + assertEquals(expected, actual); + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json new file mode 100644 index 000000000000..fa0c7a8c2e96 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json @@ -0,0 +1,38 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ClickHouse Source Spec", + "type": "object", + "required": ["host", "port", "database", "username"], + "additionalProperties": false, + "properties": { + "host": { + "description": "Host Endpoint of the Clickhouse Cluster", + "type": "string" + }, + "port": { + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 8123, + "examples": ["8123"] + }, + "database": { + "description": "Name of the database.", + "type": "string", + "examples": ["default"] + }, + "username": { + "description": "Username to use to access the database.", + "type": "string" + }, + "password": { + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index 7d3fea232919..d5feed250b5b 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -65,6 +65,10 @@ protected Map> discoverPrimaryKeys(JdbcDatabase database, private static final Logger LOGGER = LoggerFactory.getLogger(ClickHouseSource.class); public static final String DRIVER_CLASS = "ru.yandex.clickhouse.ClickHouseDriver"; + public static Source getWrappedSource() { + return new ClickHouseSource(); + } + /** * The reason we use NoOpJdbcStreamingQueryConfiguration(not setting auto commit to false and not * setting fetch size to 1000) for ClickHouse is cause method @@ -100,7 +104,7 @@ public Set getExcludedInternalNameSpaces() { } public static void main(String[] args) throws Exception { - final Source source = new ClickHouseSource(); + final Source source = ClickHouseSource.getWrappedSource(); LOGGER.info("starting source: {}", ClickHouseSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", ClickHouseSource.class); From bf97f7b39b8e0612674bd9f1e42a81c0d0ee8659 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 18 Oct 2021 18:20:36 +0300 Subject: [PATCH 03/11] [Ticket 6435] Source-ClickHouse: added ssl support --- .../connectors/source-clickhouse-strict-sncrypt/ReadMe.md | 2 +- .../ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java | 2 +- airbyte-integrations/connectors/source-clickhouse/ReadMe.md | 2 +- .../sources/SslClickHouseJdbcSourceAcceptanceTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md index 10ce40eeeb75..e490d6b7fce9 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md @@ -1,5 +1,5 @@ for ssl test custom image is used. To push it use -docker build -t airbyte/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile +docker build -t etsybaev/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile . under the tools\integration-tests-ssl dir \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index c009358bd9da..54ab1fe19334 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -85,7 +85,7 @@ public String primaryKeyClause(final List columns) { @Override @BeforeEach public void setup() throws Exception { - db = new GenericContainer("airbyte/clickhouse-with-ssl:dev").withExposedPorts(8443); + db = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md index 10ce40eeeb75..e490d6b7fce9 100644 --- a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md +++ b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md @@ -1,5 +1,5 @@ for ssl test custom image is used. To push it use -docker build -t airbyte/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile +docker build -t etsybaev/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile . under the tools\integration-tests-ssl dir \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index 4870b2f62840..55e39573f630 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -76,7 +76,7 @@ public String primaryKeyClause(final List columns) { @Override @BeforeEach public void setup() throws Exception { - db = new GenericContainer("airbyte/clickhouse-with-ssl:dev").withExposedPorts(8443); + db = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() From a39308ffa204b408ab657297d139124c78587d68 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 18 Oct 2021 22:23:40 +0300 Subject: [PATCH 04/11] Updated checkstyle --- .../integrations/source/clickhouse/ClickHouseSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index d5feed250b5b..96e5df9c196c 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -80,7 +80,7 @@ public ClickHouseSource() { } @Override - public JsonNode toDatabaseConfig(JsonNode config) { + public JsonNode toDatabaseConfig(final JsonNode config) { final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse://%s:%s/%s", config.get("host").asText(), config.get("port").asText(), @@ -103,7 +103,7 @@ public Set getExcludedInternalNameSpaces() { return Collections.singleton("system"); } - public static void main(String[] args) throws Exception { + public static void main(final String[] args) throws Exception { final Source source = ClickHouseSource.getWrappedSource(); LOGGER.info("starting source: {}", ClickHouseSource.class); new IntegrationRunner(source).run(args); From 70c0636ecbe86be6306e22b03b6115abc43ba572 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Tue, 19 Oct 2021 18:18:41 +0300 Subject: [PATCH 05/11] Fixed merge conflict, updated docs and fix typo in module's name --- .../.dockerignore | 0 .../Dockerfile | 4 ++-- .../source-clickhouse-strict-encrypt/ReadMe.md | 9 +++++++++ .../acceptance-test-config.yml | 2 +- .../build.gradle | 2 +- .../source/clickhouse/ClickHouseStrictEncryptSource.java | 0 .../ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java | 0 .../src/test-integration/resources/expected_spec.json | 0 .../source-clickhouse-strict-sncrypt/ReadMe.md | 5 ----- .../connectors/source-clickhouse/ReadMe.md | 8 +++----- docs/integrations/sources/clickhouse.md | 6 ++++++ 11 files changed, 22 insertions(+), 14 deletions(-) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/.dockerignore (100%) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/Dockerfile (64%) create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/acceptance-test-config.yml (81%) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/build.gradle (96%) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java (100%) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java (100%) rename airbyte-integrations/connectors/{source-clickhouse-strict-sncrypt => source-clickhouse-strict-encrypt}/src/test-integration/resources/expected_spec.json (100%) delete mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/.dockerignore similarity index 100% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/.dockerignore rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/.dockerignore diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile similarity index 64% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index d5deee1c6c8b..17321f6f9767 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -2,11 +2,11 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte -ENV APPLICATION source-clickhouse-strict-sncrypt +ENV APPLICATION source-clickhouse-strict-encrypt COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-clickhouse-strict-sncrypt +LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md new file mode 100644 index 000000000000..f924484ee4b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md @@ -0,0 +1,9 @@ +# Clickhouse Strict Encrypt Test Configuration + +In order to test the Clickhouse destination, you need to have the up and running Clickhouse database that has SSL enabled. + +This connector inherits the Clickhouse source, but support SSL connections only. + +# Integration tests +For ssl test custom image is used. To push it run this command under the tools\integration-tests-ssl dir: +*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml similarity index 81% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml index e817af1e1929..6755ea3b92c0 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml @@ -1,6 +1,6 @@ # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-clickhouse-strict-sncrypt:dev +connector_image: airbyte/source-clickhouse-strict-encrypt:dev tests: spec: - spec_path: "src/test-integration/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle similarity index 96% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle index 2aab9fe0f5aa..dc10ba70dbfb 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle @@ -22,7 +22,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse-strict-sncrypt') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse-strict-encrypt') integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) integrationTestJavaImplementation "org.testcontainers:clickhouse:1.16.0" } diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java similarity index 100% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java similarity index 100% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json similarity index 100% rename from airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/src/test-integration/resources/expected_spec.json rename to airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md deleted file mode 100644 index e490d6b7fce9..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse-strict-sncrypt/ReadMe.md +++ /dev/null @@ -1,5 +0,0 @@ -for ssl test custom image is used. To push it use -docker build -t etsybaev/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile . - - -under the tools\integration-tests-ssl dir \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md index e490d6b7fce9..3c38c069c587 100644 --- a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md +++ b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md @@ -1,5 +1,3 @@ -for ssl test custom image is used. To push it use -docker build -t etsybaev/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile . - - -under the tools\integration-tests-ssl dir \ No newline at end of file +# Integration tests +For ssl test custom image is used. To push it run this command under the tools\integration-tests-ssl dir: +*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* \ No newline at end of file diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index b8354651f238..783e06b5097b 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -61,3 +61,9 @@ Your database user should now be ready for use with Airbyte. | :--- | :--- | :--- | :--- | | 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | + +## CHANGELOG source-clickhouse-strict-encrypt + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | \ No newline at end of file From d897d6d0008fb1cb3863ec15682179ab0b966487 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Tue, 19 Oct 2021 18:45:19 +0300 Subject: [PATCH 06/11] updated docs --- docs/integrations/sources/clickhouse.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 783e06b5097b..0634d2c70b50 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -59,7 +59,8 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.1.3 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added SSL connections support. | +| 0.1.2 | 13.08.2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator. | ## CHANGELOG source-clickhouse-strict-encrypt From fd4626f7cb74788c962121cdbc62922b32760203 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Thu, 21 Oct 2021 13:35:05 +0300 Subject: [PATCH 07/11] Fixed tests as per comments in PR --- ...StrictEncryptJdbcSourceAcceptanceTest.java | 75 ++++++++++++------ ...SslClickHouseJdbcSourceAcceptanceTest.java | 79 +++++++++++++------ 2 files changed, 103 insertions(+), 51 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 54ab1fe19334..1884b6458960 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -7,27 +7,33 @@ import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.string.Strings; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.protocol.models.ConnectorSpecification; -import java.sql.SQLException; import java.util.List; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final String SCHEMA_NAME = "default"; - private GenericContainer db; + private static GenericContainer container; + private static JdbcDatabase db; private JsonNode config; + private String dbName; @Override public boolean supportsSchemas() { @@ -51,19 +57,55 @@ public String createTableQuery(final String tableName, // ClickHouse requires Engine to be mentioned as part of create table query. // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information return String.format("CREATE TABLE %s(%s) %s", - tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + dbName + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + primaryKeyClause); } - @Override + @BeforeAll + static void init() { + container = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); + container.start(); + } + + @BeforeEach + public void setup() throws Exception { + final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("username", "default") + .put("password", "") + .build()); + + db = Databases.createJdbcDatabase( + configWithoutDbName.get("username").asText(), + configWithoutDbName.get("password").asText(), + String.format("jdbc:clickhouse://%s:%s?ssl=true&sslmode=none", + configWithoutDbName.get("host").asText(), + configWithoutDbName.get("port").asText()), + ClickHouseSource.DRIVER_CLASS); + + dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); + + db.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); + config = Jsons.clone(configWithoutDbName); + ((ObjectNode) config).put("database", dbName); + + super.setup(); + } + @AfterEach - public void tearDown() throws SQLException { - db.close(); - db.stop(); + public void tearDownMySql() throws Exception { + db.execute(ctx -> ctx.createStatement().execute(String.format("DROP DATABASE %s;", dbName))); super.tearDown(); } + @AfterAll + public static void cleanUp() throws Exception { + db.close(); + container.close(); + } + @Override public String primaryKeyClause(final List columns) { if (columns.isEmpty()) { @@ -82,23 +124,6 @@ public String primaryKeyClause(final List columns) { return clause.toString(); } - @Override - @BeforeEach - public void setup() throws Exception { - db = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); - db.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) - .put("database", SCHEMA_NAME) - .put("username", "default") - .put("password", "") - .build()); - - super.setup(); - } - @Override public AbstractJdbcSource getJdbcSource() { return new ClickHouseSource(); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index 55e39573f630..c9f9f453d631 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -5,22 +5,28 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import java.sql.SQLException; import java.util.List; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.containers.GenericContainer; public class SslClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final String SCHEMA_NAME = "default"; - private GenericContainer db; + private static GenericContainer container; + private static JdbcDatabase db; private JsonNode config; + private String dbName; @Override public boolean supportsSchemas() { @@ -38,23 +44,61 @@ public String getDriverClass() { } @Override - public String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { + public String createTableQuery(final String tableName, + final String columnClause, + final String primaryKeyClause) { // ClickHouse requires Engine to be mentioned as part of create table query. // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information return String.format("CREATE TABLE %s(%s) %s", - tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + dbName + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + primaryKeyClause); } - @Override + @BeforeAll + static void init() { + container = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); + container.start(); + } + + @BeforeEach + public void setup() throws Exception { + final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("username", "default") + .put("password", "") + .build()); + + db = Databases.createJdbcDatabase( + configWithoutDbName.get("username").asText(), + configWithoutDbName.get("password").asText(), + String.format("jdbc:clickhouse://%s:%s?ssl=true&sslmode=none", + configWithoutDbName.get("host").asText(), + configWithoutDbName.get("port").asText()), + ClickHouseSource.DRIVER_CLASS); + + dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); + + db.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); + config = Jsons.clone(configWithoutDbName); + ((ObjectNode) config).put("database", dbName); + + super.setup(); + } + @AfterEach - public void tearDown() throws SQLException { - db.close(); - db.stop(); + public void tearDownMySql() throws Exception { + db.execute(ctx -> ctx.createStatement().execute(String.format("DROP DATABASE %s;", dbName))); super.tearDown(); } + @AfterAll + public static void cleanUp() throws Exception { + db.close(); + container.close(); + } + @Override public String primaryKeyClause(final List columns) { if (columns.isEmpty()) { @@ -73,23 +117,6 @@ public String primaryKeyClause(final List columns) { return clause.toString(); } - @Override - @BeforeEach - public void setup() throws Exception { - db = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); - db.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) - .put("database", SCHEMA_NAME) - .put("username", "default") - .put("password", "") - .build()); - - super.setup(); - } - @Override public AbstractJdbcSource getJdbcSource() { return new ClickHouseSource(); From 9dd1b18b21598e5c53d92ee52eeceb1a3780da55 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 25 Oct 2021 14:02:11 +0300 Subject: [PATCH 08/11] [7213] Source-Clickhouse: added connection via ssh support --- .../base/ssh/SshBastionContainer.java | 6 +- ...StrictEncryptJdbcSourceAcceptanceTest.java | 5 +- .../source/clickhouse/ClickHouseSource.java | 3 +- ...ractSshClickHouseSourceAcceptanceTest.java | 156 ++++++++++++++++++ .../ClickHouseSourceAcceptanceTest.java | 3 +- .../SshKeyClickhouseSourceAcceptanceTest.java | 17 ++ ...asswordClickhouseSourceAcceptanceTest.java | 17 ++ 7 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java index a455113cbc4b..a9422a3671bf 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java @@ -54,6 +54,10 @@ public JsonNode getTunnelConfig(final SshTunnel.TunnelMethod tunnelMethod, final } public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db) { + return getBasicDbConfigBuider(db, db.getDatabaseName()); + } + + public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db, final String schemaName) { return ImmutableMap.builder() .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() .getNetworks() @@ -62,7 +66,7 @@ public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDat .put("username", db.getUsername()) .put("password", db.getPassword()) .put("port", db.getExposedPorts().get(0)) - .put("database", db.getDatabaseName()) + .put("database", schemaName) .put("ssl", false); } diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 1884b6458960..a9377cd2a05d 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -138,8 +139,8 @@ public Source getSource() { void testSpec() throws Exception { final ConnectorSpecification actual = source.spec(); final ConnectorSpecification expected = - Jsons.deserialize(MoreResources.readResource("expected_spec.json"), - ConnectorSpecification.class); + SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), + ConnectorSpecification.class)); assertEquals(expected, actual); } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index 8e216ebc4dc0..0475a8304f29 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -11,6 +11,7 @@ import io.airbyte.db.jdbc.NoOpJdbcStreamingQueryConfiguration; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshWrappedSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; @@ -66,7 +67,7 @@ protected Map> discoverPrimaryKeys(final JdbcDatabase datab public static final String DRIVER_CLASS = "ru.yandex.clickhouse.ClickHouseDriver"; public static Source getWrappedSource() { - return new ClickHouseSource(); + return new SshWrappedSource(new ClickHouseSource(), List.of("host"), List.of("port")); } /** diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java new file mode 100644 index 000000000000..ea3aa549bfa5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.source.clickhouse.ClickHouseSource; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import org.jooq.SQLDialect; +import org.testcontainers.containers.ClickHouseContainer; + +public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { + + private ClickHouseContainer db; + private final SshBastionContainer bastion = new SshBastionContainer(); + private static JsonNode config; + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = "starships"; + private static final String SCHEMA_NAME = "default"; + + public abstract SshTunnel.TunnelMethod getTunnelMethod(); + + @Override + protected String getImageName() { + return "airbyte/source-clickhouse:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", config.get("database").asText(), STREAM_NAME), + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", config.get("database").asText(), STREAM_NAME2), + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + startTestContainers(); + config = bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, "default")); + populateDatabaseTestData(); + + } + private void startTestContainers() { + bastion.initAndStartBastion(); + initAndStartJdbcContainer(); + } + + private void initAndStartJdbcContainer() { + db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine").withNetwork(bastion.getNetWork()); + db.start(); + } + + private static void populateDatabaseTestData() throws Exception { + final JdbcDatabase database = Databases.createJdbcDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:clickhouse://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()), + ClickHouseSource.DRIVER_CLASS); + + final String table1 = JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME, STREAM_NAME); + final String createTable1 = + String.format("CREATE TABLE IF NOT EXISTS %s (id INTEGER, name VARCHAR(200)) ENGINE = TinyLog \n", table1); + final String table2 = JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME, STREAM_NAME2); + final String createTable2 = + String.format("CREATE TABLE IF NOT EXISTS %s (id INTEGER, name VARCHAR(200)) ENGINE = TinyLog \n", table2); + database.execute(connection -> { + connection.createStatement().execute(createTable1); + connection.createStatement().execute(createTable2); + }); + + final String insertTestData = String.format("INSERT INTO %s (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');\n", table1); + final String insertTestData2 = String.format("INSERT INTO %s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');\n", table2); + database.execute(connection -> { + connection.createStatement().execute(insertTestData); + connection.createStatement().execute(insertTestData2); + }); + + database.close(); + } + + +// @Override +// protected void tearDown(final TestDestinationEnv testEnv) { +// db.close(); +// db.stop(); +// +// } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + bastion.stopAndCloseContainers(db); + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index 28d33bcdd992..2a348a556f37 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -12,6 +12,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -43,7 +44,7 @@ protected String getImageName() { @Override protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + return SshHelpers.getSpecAndInjectSsh(); } @Override diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java new file mode 100644 index 000000000000..224586996793 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.integrations.base.ssh.SshTunnel; + +public class SshKeyClickhouseSourceAcceptanceTest extends + AbstractSshClickHouseSourceAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_KEY_AUTH; + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java new file mode 100644 index 000000000000..c38804d1403b --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.integrations.base.ssh.SshTunnel; + +public class SshPasswordClickhouseSourceAcceptanceTest extends + AbstractSshClickHouseSourceAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; + } + +} From 7a981701e2b3b5b5b2e70887550d843b4fb8711a Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Tue, 26 Oct 2021 12:48:29 +0300 Subject: [PATCH 09/11] removed commented code --- .../AbstractSshClickHouseSourceAcceptanceTest.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index ea3aa549bfa5..67cc99348497 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -140,14 +140,6 @@ private static void populateDatabaseTestData() throws Exception { database.close(); } - -// @Override -// protected void tearDown(final TestDestinationEnv testEnv) { -// db.close(); -// db.stop(); -// -// } - @Override protected void tearDown(final TestDestinationEnv testEnv) { bastion.stopAndCloseContainers(db); From f21db77f925e7b9d1042ef6b04481293a324a746 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Tue, 26 Oct 2021 19:11:55 +0300 Subject: [PATCH 10/11] Bumped versions --- .../bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../Dockerfile | 2 +- .../connectors/source-clickhouse/Dockerfile | 2 +- docs/integrations/sources/clickhouse.md | 25 +++++++++++++++++-- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json index 387a8c4195d5..999f42ca9690 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "bad83517-5e54-4a3d-9b53-63e85fbd4d7c", "name": "ClickHouse", "dockerRepository": "airbyte/source-clickhouse", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b5e793a8f304..b91e4111252d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -368,7 +368,7 @@ - sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c name: ClickHouse dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse sourceType: database - sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 17321f6f9767..7c8f406c82c0 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 024ab4bef5f1..71d03598f5fa 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 0634d2c70b50..5a22eb5b470d 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -18,8 +18,8 @@ The ClickHouse source does not alter the schema present in your warehouse. Depen | Incremental Sync | Yes | | | Replicate Incremental Deletes | Coming soon | | | Logical Replication \(WAL\) | Coming soon | | -| SSL Support | No | | -| SSH Tunnel Connection | Coming soon | | +| SSL Support | Yes | | +| SSH Tunnel Connection | Yes | | | Namespaces | Yes | Enabled by default | ## Getting started @@ -55,10 +55,30 @@ You can limit this grant down to specific tables instead of the whole database. Your database user should now be ready for use with Airbyte. +## Connection via SSH Tunnel + +Airbyte has the ability to connect to a Clickhouse instance via an SSH Tunnel. The reason you might want to do this because it is not possible \(or against security policy\) to connect to the database directly \(e.g. it does not have a public IP address\). + +When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server \(a.k.a. a bastion sever\) that _does_ have direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. + +Using this feature requires additional configuration, when creating the source. We will talk through what each piece of configuration means. + +1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. +2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. + 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\). + 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. +3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address. +4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default. +5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the Clickhouse username. +6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the Clickhouse password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. +7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. + + ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | | 0.1.3 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added SSL connections support. | | 0.1.2 | 13.08.2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator. | @@ -67,4 +87,5 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | | 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | \ No newline at end of file From eff7b5c65a598a956f31fb60a45f4aa2d87f941d Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Tue, 26 Oct 2021 19:59:29 +0300 Subject: [PATCH 11/11] fixed merge conflict --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index ded5a1878541..5c524118b490 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -87,7 +87,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse sourceType: database - name: Close.com