From 4790588009b4794bfdd61c1623544858d778f42e Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Thu, 14 Jul 2022 10:01:14 +0700 Subject: [PATCH 01/15] 13539 Fix integration tests source-clickhouse Mac OS --- .../sources/AbstractSshClickHouseSourceAcceptanceTest.java | 2 +- .../sources/ClickHouseJdbcSourceAcceptanceTest.java | 5 +++-- .../sources/ClickHouseJdbcStressTest.java | 7 ++++--- .../sources/ClickHouseSourceAcceptanceTest.java | 2 +- .../sources/SslClickHouseJdbcSourceAcceptanceTest.java | 5 +++-- deps.toml | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index dc1ca2afa2b1..af0d9f799332 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -100,7 +100,7 @@ private void startTestContainers() { } private void initAndStartJdbcContainer() { - db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine").withNetwork(network); + db = (ClickHouseContainer) new ClickHouseContainer("clickhouse/clickhouse-server:22.5").withNetwork(network); db.start(); } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 456e9ec9bacf..79718cf9ce30 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -10,6 +10,7 @@ import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.sql.SQLException; import java.util.List; @@ -81,8 +82,8 @@ public void setup() throws Exception { db.start(); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) + .put("host", HostPortResolver.resolveHost(db)) + .put("port", HostPortResolver.resolvePort(db)) .put("database", SCHEMA_NAME) .put("username", db.getUsername()) .put("password", db.getPassword()) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java index 6e9ad0c59521..5d95af5b7a65 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java @@ -10,6 +10,7 @@ import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.util.Optional; import org.junit.jupiter.api.AfterEach; @@ -32,12 +33,12 @@ public Optional getDefaultSchemaName() { @Override @BeforeEach public void setup() throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) + .put("host", HostPortResolver.resolveHost(db)) + .put("port", HostPortResolver.resolvePort(db)) .put("database", SCHEMA_NAME) .put("username", db.getUsername()) .put("password", db.getPassword()) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index 716103a4a386..fc629c4abb15 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -82,7 +82,7 @@ protected JsonNode getState() { @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index b12a984f2186..1cf7a313837c 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.util.List; import javax.sql.DataSource; @@ -68,8 +69,8 @@ static void init() { @BeforeEach public void setup() throws Exception { final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) + .put("host", HostPortResolver.resolveHost(container)) + .put("port", HostPortResolver.resolvePort(container)) .put("username", "default") .put("password", "") .build()); diff --git a/deps.toml b/deps.toml index fb062d43f948..3eb6a0163e8e 100644 --- a/deps.toml +++ b/deps.toml @@ -18,7 +18,7 @@ connectors-testcontainers-scylla = "1.16.2" connectors-testcontainers-tidb = "1.16.3" connectors-destination-testcontainers-clickhouse = "1.17.3" connectors-destination-testcontainers-oracle-xe = "1.16.0" -connectors-source-testcontainers-clickhouse = "1.16.0" +connectors-source-testcontainers-clickhouse = "1.17.3" platform-testcontainers = "1.17.1" [libraries] From 3f366c4f65a6115454805c9fd6f70dcedcdf79f6 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Wed, 20 Jul 2022 14:16:29 +0700 Subject: [PATCH 02/15] 13539 Updated clickhouse jdbc driver 13539 Added new config for clickhouse source ssl --- .../io/airbyte/db/factory/DatabaseDriver.java | 2 +- .../destination-clickhouse/build.gradle | 3 +- .../clickhouse/ClickhouseDestination.java | 8 ++-- .../clickhouse/ClickhouseSqlOperations.java | 24 +++++----- .../ClickhouseDestinationAcceptanceTest.java | 9 +++- ...shClickhouseDestinationAcceptanceTest.java | 1 + .../clickhouse/ClickhouseDestinationTest.java | 6 +-- .../connectors/source-clickhouse/build.gradle | 2 +- .../source/clickhouse/ClickHouseSource.java | 17 +++---- ...ractSshClickHouseSourceAcceptanceTest.java | 10 +++- .../ClickHouseJdbcSourceAcceptanceTest.java | 8 +++- .../sources/ClickHouseJdbcStressTest.java | 8 +++- .../ClickHouseSourceAcceptanceTest.java | 9 +++- ...SslClickHouseJdbcSourceAcceptanceTest.java | 47 ++++++++++++------- .../resources/docker/Dockerfile | 8 ++++ .../resources/docker}/clickhouse_certs.sh | 3 -- .../test-integration/resources/ssl_ports.xml | 26 ++++++++++ .../Clickhouse.Dockerfile | 9 ---- 18 files changed, 136 insertions(+), 64 deletions(-) create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile rename {tools/integrations-test-ssl => airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker}/clickhouse_certs.sh (79%) create mode 100644 airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/ssl_ports.xml delete mode 100644 tools/integrations-test-ssl/Clickhouse.Dockerfile diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java index deff79c20e38..eee395a4e2e9 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java @@ -9,7 +9,7 @@ */ public enum DatabaseDriver { - CLICKHOUSE("ru.yandex.clickhouse.ClickHouseDriver", "jdbc:clickhouse://%s:%d/%s"), + CLICKHOUSE("com.clickhouse.jdbc.ClickHouseDriver", "jdbc:clickhouse:%s://%s:%d/%s"), DATABRICKS("com.databricks.client.jdbc.Driver", "jdbc:databricks://%s;HttpPath=%s;UserAgentEntry=Airbyte"), DB2("com.ibm.db2.jcc.DB2Driver", "jdbc:db2://%s:%d/%s"), MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"), diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 8400dc16962b..c72e1a9d6854 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -17,8 +17,7 @@ dependencies { implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - // https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc - implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1-patch' + implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse testImplementation libs.connectors.destination.testcontainers.clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index fcdd4a0d67bb..9b0fdc24a4ed 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -17,7 +17,6 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import java.util.HashMap; import java.util.List; import java.util.Map; import javax.sql.DataSource; @@ -32,12 +31,13 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + public static final String HTTPS_PROTOCOL = "https"; + public static final String HTTP_PROTOCOL = "http"; private static final String PASSWORD = "password"; static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "socket_timeout", "3000000", - "ssl", "true", "sslmode", "none"); public static Destination sshWrappedDestination() { @@ -50,7 +50,9 @@ public ClickhouseDestination() { @Override public JsonNode toJdbcConfig(final JsonNode config) { - final String jdbcUrl = String.format("jdbc:clickhouse://%s:%s/%s?", + final boolean isSsl = useSsl(config); + final String jdbcUrl = String.format("jdbc:clickhouse:%s://%s:%s/%s", + isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asText(), config.get("database").asText()); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 09c48a7bb50b..73570f231e61 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.destination.clickhouse; +import com.clickhouse.client.ClickHouseFile; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseRequest; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; @@ -12,12 +15,10 @@ import java.io.IOException; import java.nio.file.Files; import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ru.yandex.clickhouse.ClickHouseConnection; -import ru.yandex.clickhouse.ClickHouseStatement; -import ru.yandex.clickhouse.domain.ClickHouseFormat; public class ClickhouseSqlOperations extends JdbcSqlOperations { @@ -52,11 +53,10 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN @Override public void executeTransaction(final JdbcDatabase database, final List queries) throws Exception { - final StringBuilder appendedQueries = new StringBuilder(); + // Note: ClickHouse does not support multi query for (final String query : queries) { - appendedQueries.append(query); + database.execute(query); } - database.execute(appendedQueries.toString()); } @Override @@ -77,12 +77,12 @@ public void insertRecordsInternal(final JdbcDatabase database, try { tmpFile = Files.createTempFile(tmpTableName + "-", ".tmp").toFile(); writeBatchToFile(tmpFile, records); - - ClickHouseConnection conn = connection.unwrap(ClickHouseConnection.class); - ClickHouseStatement sth = conn.createStatement(); - sth.write() // Write API entrypoint - .table(String.format("%s.%s", schemaName, tmpTableName)) // where to write data - .data(tmpFile, ClickHouseFormat.CSV) // specify input + Statement sth = connection.createStatement(); + sth.unwrap(ClickHouseRequest.class) + .write() + .table(String.format("%s.%s", schemaName, tmpTableName)) + .data(ClickHouseFile.of(tmpFile)) + .format(ClickHouseFormat.CSV) .send(); } catch (final Exception e) { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 45a245664663..840806cab864 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.clickhouse; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; @@ -18,10 +20,12 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.wait.strategy.Wait; public class ClickhouseDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -135,6 +139,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) { config.has("password") ? config.get("password").asText() : null, ClickhouseDestination.DRIVER_CLASS, String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + ClickhouseDestination.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asInt(), config.get("database").asText()))); @@ -142,7 +147,9 @@ private static JdbcDatabase getDatabase(final JsonNode config) { @Override protected void setup(final TestDestinationEnv testEnv) { - db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 4cf2e5b4d589..2e86ed40281b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -153,6 +153,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) { config.has("password") ? config.get("password").asText() : null, ClickhouseDestination.DRIVER_CLASS, String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + ClickhouseDestination.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asInt(), config.get("database").asText()))); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 7a7257424aec..c288edc0cfdd 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -27,7 +27,6 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.time.Instant; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -62,7 +61,7 @@ public class ClickhouseDestinationTest { @BeforeAll static void init() { - db = new ClickHouseContainer("yandex/clickhouse-server"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); db.start(); } @@ -132,7 +131,8 @@ void sanityTest() throws Exception { config.get("username").asText(), config.get("password").asText(), ClickhouseDestination.DRIVER_CLASS, - String.format("jdbc:clickhouse://%s:%s/%s", + String.format("jdbc:clickhouse:%s://%s:%s/%s", + ClickhouseDestination.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asText(), config.get("database").asText()))); diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index 5280ad611802..e15b1e743883 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-protocol:protocol-models') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1' + implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index 48991668dff1..3142ee66d6e4 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -20,7 +20,6 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,9 +37,9 @@ public class ClickHouseSource extends AbstractJdbcSource implements So * https://clickhouse.tech/docs/en/operations/system-tables/columns/ to fetch the primary keys. */ - public static final List SSL_PARAMETERS = List.of( - "ssl=true", - "sslmode=none"); + public static final String SSL_MODE = "sslmode=none"; + public static final String HTTPS_PROTOCOL = "https"; + public static final String HTTP_PROTOCOL = "http"; @Override protected Map> discoverPrimaryKeys(final JdbcDatabase database, @@ -82,14 +81,16 @@ public ClickHouseSource() { @Override public JsonNode toDatabaseConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse://%s:%s/%s", + boolean isSsl = !config.has("ssl") || config.get("ssl").asBoolean(); + final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse:%s://%s:%s/%s", + isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asText(), config.get("database").asText())); // assume ssl if not explicitly mentioned. - if (!config.has("ssl") || config.get("ssl").asBoolean()) { - jdbcUrl.append("?").append(String.join("&", SSL_PARAMETERS)); + if (isSsl) { + jdbcUrl.append("?").append(SSL_MODE); } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() @@ -105,7 +106,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { @Override public Set getExcludedInternalNameSpaces() { - return Collections.singleton("system"); + return Set.of("system", "information_schema", "INFORMATION_SCHEMA"); } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index af0d9f799332..954f6438326e 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; @@ -26,10 +28,12 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; +import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { @@ -100,7 +104,10 @@ private void startTestContainers() { } private void initAndStartJdbcContainer() { - db = (ClickHouseContainer) new ClickHouseContainer("clickhouse/clickhouse-server:22.5").withNetwork(network); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .withNetwork(network) + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); } @@ -110,6 +117,7 @@ private static void populateDatabaseTestData() throws Exception { config.get("password").asText(), ClickHouseSource.DRIVER_CLASS, String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + ClickHouseSource.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asInt(), config.get("database").asText())); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 79718cf9ce30..51a281109099 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; @@ -13,10 +15,12 @@ import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.wait.strategy.Wait; public class ClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { @@ -78,7 +82,9 @@ public String primaryKeyClause(final List columns) { @Override @BeforeEach public void setup() throws Exception { - db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java index 5d95af5b7a65..68569e49d2e2 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; @@ -12,11 +14,13 @@ import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; +import java.time.Duration; import java.util.Optional; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.wait.strategy.Wait; @Disabled public class ClickHouseJdbcStressTest extends JdbcStressTest { @@ -33,7 +37,9 @@ public Optional getDefaultSchemaName() { @Override @BeforeEach public void setup() throws Exception { - db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index fc629c4abb15..b92f4b472f60 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -25,9 +27,11 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; +import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.wait.strategy.Wait; public class ClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { @@ -82,7 +86,9 @@ protected JsonNode getState() { @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5"); + db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); config = Jsons.jsonNode(ImmutableMap.builder() @@ -99,6 +105,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get("password").asText(), ClickHouseSource.DRIVER_CLASS, String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + ClickHouseSource.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asInt(), config.get("database").asText())); diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index 1cf7a313837c..4807b3ff44ce 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; @@ -15,15 +17,19 @@ import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; +import java.io.IOException; import java.sql.JDBCType; +import java.time.Duration; import java.util.List; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; public class SslClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { @@ -33,6 +39,25 @@ public class SslClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceT private JsonNode config; private String dbName; + @BeforeAll + static void init() throws IOException, InterruptedException { + container = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") + .withFileFromClasspath("Dockerfile", "docker/Dockerfile") + .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) + .withEnv("TZ", "UTC") + .withExposedPorts(8123, 8443) + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + container.start(); + } + + @AfterAll + public static void cleanUp() throws Exception { + DataSourceFactory.close(dataSource); + container.close(); + } + @Override public boolean supportsSchemas() { return false; @@ -60,17 +85,11 @@ public String createTableQuery(final String tableName, + primaryKeyClause); } - @BeforeAll - static void init() { - container = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); - container.start(); - } - @BeforeEach public void setup() throws Exception { final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put("host", HostPortResolver.resolveHost(container)) - .put("port", HostPortResolver.resolvePort(container)) + .put("host", "localhost") + .put("port", container.getMappedPort(8443)) .put("username", "default") .put("password", "") .build()); @@ -81,7 +100,7 @@ public void setup() throws Exception { config.get("username").asText(), config.get("password").asText(), ClickHouseSource.DRIVER_CLASS, - String.format("jdbc:clickhouse://%s:%d?ssl=true&sslmode=none", + String.format("jdbc:clickhouse:https://%s:%d?sslmode=NONE", config.get("host").asText(), config.get("port").asInt())); @@ -96,17 +115,11 @@ public void setup() throws Exception { } @AfterEach - public void tearDownMySql() throws Exception { + public void tearDownClickHouse() throws Exception { jdbcDatabase.execute(ctx -> ctx.createStatement().execute(String.format("DROP DATABASE %s;", dbName))); super.tearDown(); } - @AfterAll - public static void cleanUp() throws Exception { - DataSourceFactory.close(dataSource); - container.close(); - } - @Override public String primaryKeyClause(final List columns) { if (columns.isEmpty()) { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile new file mode 100644 index 000000000000..856204a987cb --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM clickhouse/clickhouse-server:22.5 + +EXPOSE 8123 +EXPOSE 8443 + +RUN apt-get update && apt-get install -y net-tools curl + +COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ diff --git a/tools/integrations-test-ssl/clickhouse_certs.sh b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/clickhouse_certs.sh similarity index 79% rename from tools/integrations-test-ssl/clickhouse_certs.sh rename to airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/clickhouse_certs.sh index ab692baf9bcd..285a2bc58c54 100644 --- a/tools/integrations-test-ssl/clickhouse_certs.sh +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/clickhouse_certs.sh @@ -9,7 +9,4 @@ openssl dhparam -out /etc/clickhouse-server/dhparam.pem 1024 chown $(id -u clickhouse):$(id -g clickhouse) /etc/clickhouse-server/server.{key,crt} -echo "8443" > /etc/clickhouse-server/config.d/https.xml - - echo "Finished preparing certs" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/ssl_ports.xml b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/ssl_ports.xml new file mode 100644 index 000000000000..82018f7eae6e --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/ssl_ports.xml @@ -0,0 +1,26 @@ + + + 8123 + 8443 + 9000 + 9440 + 9004 + 9005 + 9009 + 9100 + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + relaxed + + ConsoleCertificateHandler + + false + true + sslv2,sslv3 + true + + + \ No newline at end of file diff --git a/tools/integrations-test-ssl/Clickhouse.Dockerfile b/tools/integrations-test-ssl/Clickhouse.Dockerfile deleted file mode 100644 index a4288be5af09..000000000000 --- a/tools/integrations-test-ssl/Clickhouse.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM yandex/clickhouse-server:latest - -EXPOSE 8443 -EXPOSE 9000 -EXPOSE 8123 -EXPOSE 9009 - -COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ - From 860a68f3492cd37392530d7cf22ee19d10bb65e9 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Wed, 20 Jul 2022 14:44:09 +0700 Subject: [PATCH 03/15] 13539 Bump up the versions --- .../connectors/destination-clickhouse/Dockerfile | 2 +- airbyte-integrations/connectors/source-clickhouse/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index e01aa6d0ca51..c7d1bfd9ff90 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 1f9b1623150c..9f5553e81281 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-clickhouse From 4a0b97a74081eda757c03e9bb1b9522ed686b324 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Wed, 20 Jul 2022 15:09:17 +0700 Subject: [PATCH 04/15] 13539 Fixed merge conflicts --- .../destination/clickhouse/ClickhouseDestination.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 52f934f9963c..c25cf640cda8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -103,6 +103,10 @@ protected Map getDefaultConnectionProperties(final JsonNode conf } } + private boolean useSsl(final JsonNode config) { + return !config.has("ssl") || config.get("ssl").asBoolean(); + } + public static void main(final String[] args) throws Exception { final Destination destination = ClickhouseDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", ClickhouseDestination.class); From 9c121f98e74dfe2e84aa7e02b666cb6e2d839732 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Thu, 21 Jul 2022 14:40:15 +0700 Subject: [PATCH 05/15] 13539 Updated destination-clickhouse-strict-encrypt --- .../build.gradle | 3 +- ...estinationStrictEncryptAcceptanceTest.java | 38 +- .../src/test-integration/resources/config.xml | 1161 ----------------- .../test-integration/resources/dhparam.pem | 13 - .../resources/docker/Dockerfile | 6 + .../resources/docker/clickhouse_certs.sh | 12 + .../src/test-integration/resources/server.crt | 19 - .../src/test-integration/resources/server.key | 28 - .../test-integration/resources/ssl_ports.xml | 26 + .../clickhouse/ClickhouseDestination.java | 7 +- 10 files changed, 71 insertions(+), 1242 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/config.xml delete mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/dhparam.pem create mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile create mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh delete mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.crt delete mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.key create mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index 8348797d544d..31b2bcf7241e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -17,8 +17,7 @@ dependencies { implementation project(':airbyte-integrations:connectors:destination-clickhouse') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - // https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc - implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1-patch' + implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse testImplementation libs.connectors.destination.testcontainers.clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 92c06a253e25..8c331b1e44d8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -4,17 +4,21 @@ package io.airbyte.integrations.destination.clickhouse; +import static java.time.temporal.ChronoUnit.SECONDS; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -22,7 +26,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; public class ClickhouseDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest { @@ -32,7 +38,7 @@ public class ClickhouseDestinationStrictEncryptAcceptanceTest extends Destinatio private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); - private ClickHouseContainer db; + private GenericContainer db; public static final Integer HTTP_PORT = 8123; public static final Integer NATIVE_PORT = 9000; @@ -76,8 +82,8 @@ protected JsonNode getConfig() { .put("host", db.getHost()) .put("port", db.getMappedPort(HTTPS_PORT)) .put("database", DB_NAME) - .put("username", db.getUsername()) - .put("password", db.getPassword()) + .put("username", "default") + .put("password", "") .put("schema", DB_NAME) .build()); } @@ -129,7 +135,8 @@ protected List resolveIdentifier(final String identifier) { } private static JdbcDatabase getDatabase(final JsonNode config) { - final String jdbcStr = String.format("jdbc:clickhouse://%s:%s/%s?ssl=true&sslmode=none", + final String jdbcStr = String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + ClickhouseDestination.HTTPS_PROTOCOL, config.get("host").asText(), config.get("port").asText(), config.get("database").asText()); @@ -142,22 +149,27 @@ private static JdbcDatabase getDatabase(final JsonNode config) { @Override protected void setup(final TestDestinationEnv testEnv) { - db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server") + db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") + .withFileFromClasspath("Dockerfile", "docker/Dockerfile") + .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) + .withEnv("TZ", "UTC") .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) - .withClasspathResourceMapping("config.xml", "/etc/clickhouse-server/config.xml", BindMode.READ_ONLY) - .withClasspathResourceMapping("server.crt", "/etc/clickhouse-server/server.crt", BindMode.READ_ONLY) - .withClasspathResourceMapping("server.key", "/etc/clickhouse-server/server.key", BindMode.READ_ONLY) - .withClasspathResourceMapping("dhparam.pem", "/etc/clickhouse-server/dhparam.pem", BindMode.READ_ONLY); + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + db.start(); - LOGGER.info(String.format("Clickhouse server container port mapping: %d -> %d, %d -> %d", + LOGGER.info(String.format("Clickhouse server container port mapping: %d -> %d, %d -> %d, %d -> %d, %d -> %d", HTTP_PORT, db.getMappedPort(HTTP_PORT), - HTTPS_PORT, db.getMappedPort(HTTPS_PORT))); + HTTPS_PORT, db.getMappedPort(HTTPS_PORT), + NATIVE_PORT, db.getMappedPort(NATIVE_PORT), + NATIVE_SECURE_PORT, db.getMappedPort(NATIVE_SECURE_PORT) + )); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - db.stop(); db.close(); } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/config.xml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/config.xml deleted file mode 100644 index 9b7432e4f240..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/config.xml +++ /dev/null @@ -1,1161 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - 8443 - - - 9440 - - - 9011 - - - 9009 - - - - - - - - - - - - :: - - - 0.0.0.0 - - - ::1 - 127.0.0.1 - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - - true - - - AcceptCertificateHandler - - - - - true - true - sslv2,sslv3 - true - - - - AcceptCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - - - - - - - - - localhost - 9000 - - - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - 127.0.0.1 - 9000 - - - - - 127.0.0.2 - 9000 - - - - - - true - - 127.0.0.1 - 9000 - - - - true - - 127.0.0.2 - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 -
- - - - system - trace_log
- - toYYYYMM(event_date) - 7500 -
- - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 -
- - - - - - - - system - metric_log
- 7500 - 1000 -
- - - - system - asynchronous_metric_log
- - 7000 -
- - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 -
- - - - - system - crash_log
- - - 1000 -
- - - - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - -
diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/dhparam.pem b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/dhparam.pem deleted file mode 100644 index 2a862dd18e82..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/dhparam.pem +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIICCAKCAgEArsn0Y1ZPxaUNw4QREGoCFSWUhg05CVgDIlhQ42ixrIRPW+WduZwO -KqnCohYkMQvFM0J0s4laNXhWXjrEYtnzwp81M7t/3gmXxkglyE4gk9BhpmLpv/JU -TLVhjWLggOZTsEWyiR4REUvJ1IvaQ/K9RGoOSw4tgyW8gHGLjPHxhTPtp5ZfE5TJ -OQZsByLcQbqji3jCoZeNRGcOjMPri4A0u8cXlHQr3/t49G/nE2oC296MVAZNkdjt -mDQmNX8Ej5dm6F0ZWYFptgxkJknaBDjsJh+ga8SKG6dZvqEMvTqHLTOV8h2uFei9 -Gm4DtKWf8x232s9t+aGOF+qsPss1lM8spYTnY9B6jdEzEwEiXFBogG763lJNJpLv -nsk8YofO6hJrgiKJWiSbR50qo6us1cq191mFDBWO8yWSjfXgf5HpzutO1hVJPKyS -+3VTt3ZfRFBJZozWlQzddurwd5Wr+D3JR8E9mz7YukEA7iLwM3nZLcF1b35cYCRs -9Q59ezxyveVtWAcTBBzRzNGr3mf9LYoumd5o3jFJsLLWAHfDZXey7n6tBJUh6a6N -ChOCeoavhzlv3lx2+C19ZKkTKU+Z/cCjnW9530MVFtlXFqihIaVAB/ecAF0ZsPNB -pg5I+U4TR0J684eoM4LgMo0ydq7G9g+WbgQ+aD0C7CcEAV7t8iuj9IMCAQI= ------END DH PARAMETERS----- diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile new file mode 100644 index 000000000000..ca9d01ecea13 --- /dev/null +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM clickhouse/clickhouse-server:22.5 + +EXPOSE 8123 +EXPOSE 8443 + +COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh new file mode 100644 index 000000000000..285a2bc58c54 --- /dev/null +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh @@ -0,0 +1,12 @@ +echo "Preparing certs" + +openssl req -subj "/CN=my.host.name" -new \ + -newkey rsa:2048 -days 365 -nodes -x509 \ + -keyout /etc/clickhouse-server/server.key \ + -out /etc/clickhouse-server/server.crt + +openssl dhparam -out /etc/clickhouse-server/dhparam.pem 1024 + +chown $(id -u clickhouse):$(id -g clickhouse) /etc/clickhouse-server/server.{key,crt} + +echo "Finished preparing certs" \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.crt b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.crt deleted file mode 100644 index 0aeb3cdc4027..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDCTCCAfGgAwIBAgIUcuYIRxO5jbEzvookG0MTrb2jqDUwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIxMTEwNzAzNDI0MloXDTIyMTEw -NzAzNDI0MlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF -AAOCAQ8AMIIBCgKCAQEAmKDFGjs1/TpNv7ZOB9LTeSYOIjo50An4ahYO+xTGVQJS -SajsNs2NkZWXRUe+WOGANFjTWxTGs/vYWlJ/gC/0GckcuSyqWAShVfZuYHQsIFAG -yFYTAcXQ9bJPPiNTOwq00WqnhgQZ121EGiIYnxZ8XqR1CPS4fXMDFr5lSSrlu/yM -vzRQOEbiiKcrRZvikiFijFDt759J4r3Pr4+tmZ2GJe28ZI9sMmeLigup1/awna9i -7j1yS5iHwFd0xVXSVkPdym3crNDPG9Iy2QicDqijJpqDzKErwR71839x0E8B53HF -PL6U6KtACViDklPYn+IWsAuRV9o56/X3X6tmt8YjBQIDAQABo1MwUTAdBgNVHQ4E -FgQUNoWDQBHyh0EUzpvEtbktJwvPol8wHwYDVR0jBBgwFoAUNoWDQBHyh0EUzpvE -tbktJwvPol8wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAJZLh -BdmGt85Ru8ACtrJ8P/59GjtLBXCDAyrQeR+SfPShoL3xx69ldh9flRWOgODxHQiX -tGMDPbXZ5dRUAPmwOEjDecYk5H6RXKWa1BqX5yAa1grfkZsj1+9Pv7DneQae/owq -RAVd0SWjomFL0oElUvHhk7AMpHu7XK9f3Vj2JPFcieD7SAotn6i3IFKFrBTz14T4 -6O0tgEUdcujr0DsIxZsJRt2ITXgdJCOT+ohDWTdqB5+vh+HxfaSxAPssLBwD1bFF -m3mEBNCkrFCpX/4+uYw559ah9KRAN8nVpzl1fKZLIms/6UCdA65FEGr+7m39UDa2 -leTb3J/1adaimGBHkw== ------END CERTIFICATE----- diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.key b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.key deleted file mode 100644 index 6d29bcb797fa..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/server.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYoMUaOzX9Ok2/ -tk4H0tN5Jg4iOjnQCfhqFg77FMZVAlJJqOw2zY2RlZdFR75Y4YA0WNNbFMaz+9ha -Un+AL/QZyRy5LKpYBKFV9m5gdCwgUAbIVhMBxdD1sk8+I1M7CrTRaqeGBBnXbUQa -IhifFnxepHUI9Lh9cwMWvmVJKuW7/Iy/NFA4RuKIpytFm+KSIWKMUO3vn0nivc+v -j62ZnYYl7bxkj2wyZ4uKC6nX9rCdr2LuPXJLmIfAV3TFVdJWQ93Kbdys0M8b0jLZ -CJwOqKMmmoPMoSvBHvXzf3HQTwHnccU8vpToq0AJWIOSU9if4hawC5FX2jnr9fdf -q2a3xiMFAgMBAAECggEABRh4/CM17jpgFiJ0AHbdcJIArsva2kM68+/AmEldRcFK -FgN5v2qla1LfNgTrKfR9X0IH7GaJK8EgvAM9Fn6DW8kHGHeRSFb/O+t6S+YVAw8o -6s9U2x5Ll6hYVO7A/Jui5cr1xNemNTcEqwqwX79Ub6R189KXxjhsxfmdUBAKk3gR -1yOsVhsN0B2FnVCF8V1Ad3G8AUtZ3Akrclex8kRSMIKYCDhhn8p1kNWQT7MSYXnD -YGJdO/a3ZdXUJvUG7W3J/lX6rS9A4dDA8S9AgQ06XroODdm+NrP2qFfseOeyZpYs -m0mRfSNasWXStU+IyjK/wuYLGS7YgeLASHReUMUWgQKBgQDKItQqpF4SxwfJCyET -W2HYcU6ho5B37EPcF7gBPFwg+/8Mr0vW1cr3HgCycZpud1T4v0Hcd0ZyPkEFZ9J1 -HAzmWyzMXyvBHrb5uUeOO3SHqOBKhhXCj+EVJUmgTjKmMMUSj2NGB5HycTYghCo8 -faGRe7knOXNBHbKayb5p8b5kSQKBgQDBTKMiF/V1D2O+j9/8Y8Voy6iBS6pMLzgF -5wk/jplTwBIxBE8RXRHkEOIdoO74I/8NvB4HSGRpN4ftoqPneGA4AUaPsvn9e76z -+7HhVCvAPCgdNMi2XIL76f0YFJPIBisnggun87hyAseUw/R6s9szlA1iQM6r94jX -S+eD5P0Q3QKBgFf9MT4erpSd7tWk8pDqhn0hVYknGgwZ3LBB4ucmzzPMfgmXhPvP -tLo9ZTBII2FSsLpXBuRhR1kToFoqB4LS4wqjRazxAKrgI+YwsOVJHECxUnEeTPqU -hvYddR9C6ulM8XxSznP9d5qBjX46CeirB1m88awRZgpCUyzuBOU5RheBAoGAWrbA -iCt+QeVrBe83Spb3+eo5thPwY7h1Li/yoyUkx60H3IENKjTnRIS32PfBrioWdDeo -T/qlRMuOuvLswKA5Z48RsjZoI5GDOawRGpIJxjl1Cd/PoeVggyCYakid4e0jK3NY -TQWPtdGgICyl+z+Uy2vbrBSF6SZNzdwNVlSMfvECgYAoPb1ihbS0+mSKvtBaw9dV -fWP0ktVEEIw5BLEeD+75goYKTTNtJ7hV47aAnEcpSOlO77I3G8E5v9YL/jlXMfvF -3wi2dfGgDbKuXCtCQS/BqnqDhw9Py3sg9XzgFdLM7Y0Z7JJHTDyV9b/PfAMk6bgn -SKOWqhX1fJgG5kgymk1YEg== ------END PRIVATE KEY----- diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml new file mode 100644 index 000000000000..82018f7eae6e --- /dev/null +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml @@ -0,0 +1,26 @@ + + + 8123 + 8443 + 9000 + 9440 + 9004 + 9005 + 9009 + 9100 + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + relaxed + + ConsoleCertificateHandler + + false + true + sslv2,sslv3 + true + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index c25cf640cda8..077b3497e1ba 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -51,7 +51,7 @@ public ClickhouseDestination() { @Override public JsonNode toJdbcConfig(final JsonNode config) { - final boolean isSsl = useSsl(config); + final boolean isSsl = JdbcUtils.useSsl(config); final String jdbcUrl = String.format("jdbc:clickhouse:%s://%s:%s/%s", isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, config.get("host").asText(), @@ -69,7 +69,6 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - @Override public AirbyteConnectionStatus check(final JsonNode config) { final DataSource dataSource = getDataSource(config); @@ -103,10 +102,6 @@ protected Map getDefaultConnectionProperties(final JsonNode conf } } - private boolean useSsl(final JsonNode config) { - return !config.has("ssl") || config.get("ssl").asBoolean(); - } - public static void main(final String[] args) throws Exception { final Destination destination = ClickhouseDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", ClickhouseDestination.class); From 105c4fc0059e5912fa37277ac903edfd390d77b3 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 22 Jul 2022 13:57:55 +0700 Subject: [PATCH 06/15] 13539 Updated SSL configuration and tests for clickhouse-destination --- .../Dockerfile | 2 +- ...estinationStrictEncryptAcceptanceTest.java | 72 +++++++++++++------ .../ClickhouseTestDataComparator.java | 29 ++++++++ .../resources/docker/Dockerfile | 2 + .../clickhouse/ClickhouseDestination.java | 30 ++++---- .../clickhouse/ClickhouseDestinationTest.java | 17 +---- .../resources/docker/Dockerfile | 4 +- 7 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index f1ec23dc5504..d64303770208 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 8c331b1e44d8..0f8e37d9ae6a 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -17,6 +17,8 @@ import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; @@ -34,16 +36,27 @@ public class ClickhouseDestinationStrictEncryptAcceptanceTest extends Destinatio private static final Logger LOGGER = LoggerFactory.getLogger(ClickhouseDestinationStrictEncryptAcceptanceTest.class); - private static final String DB_NAME = "default"; - - private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); - - private GenericContainer db; - public static final Integer HTTP_PORT = 8123; public static final Integer NATIVE_PORT = 9000; public static final Integer HTTPS_PORT = 8443; public static final Integer NATIVE_SECURE_PORT = 9440; + private static final String DB_NAME = "default"; + private static final String USER_NAME = "default"; + private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); + private GenericContainer db; + + private static JdbcDatabase getDatabase(final JsonNode config) { + final String jdbcStr = String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString() + "?sslmode=none", + ClickhouseDestination.HTTPS_PROTOCOL, + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()); + return new DefaultJdbcDatabase(DataSourceFactory.create( + config.get("username").asText(), + config.has("password") ? config.get("password").asText() : null, + ClickhouseDestination.DRIVER_CLASS, + jdbcStr)); + } @Override protected String getImageName() { @@ -52,7 +65,7 @@ protected String getImageName() { @Override protected boolean supportsNormalization() { - return true; + return false; } @Override @@ -65,6 +78,26 @@ protected boolean implementsNamespaces() { return true; } + @Override + protected TestDataComparator getTestDataComparator() { + return new ClickhouseTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + @Override protected String getDefaultSchema(final JsonNode config) { if (config.get("database") == null) { @@ -79,10 +112,10 @@ protected JsonNode getConfig() { // dbt clickhouse adapter uses native protocol, its default port is 9000 // Since we disabled normalization and dbt test, we only use the JDBC port here. return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getMappedPort(HTTPS_PORT)) + .put("host", HostPortResolver.resolveHost(db)) + .put("port", HTTPS_PORT) .put("database", DB_NAME) - .put("username", "default") + .put("username", USER_NAME) .put("password", "") .put("schema", DB_NAME) .build()); @@ -134,19 +167,6 @@ protected List resolveIdentifier(final String identifier) { return result; } - private static JdbcDatabase getDatabase(final JsonNode config) { - final String jdbcStr = String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), - ClickhouseDestination.HTTPS_PROTOCOL, - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()); - return new DefaultJdbcDatabase(DataSourceFactory.create( - config.get("username").asText(), - config.has("password") ? config.get("password").asText() : null, - ClickhouseDestination.DRIVER_CLASS, - jdbcStr)); - } - @Override protected void setup(final TestDestinationEnv testEnv) { db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") @@ -170,6 +190,7 @@ protected void setup(final TestDestinationEnv testEnv) { @Override protected void tearDown(final TestDestinationEnv testEnv) { + db.stop(); db.close(); } @@ -213,4 +234,9 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin super.testSyncWithNormalization(messagesFilename, catalogFilename); } + @Disabled + public void specNormalizationValueShouldBeCorrect() throws Exception { + super.specNormalizationValueShouldBeCorrect(); + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java new file mode 100644 index 000000000000..00e64e6256ec --- /dev/null +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.clickhouse; + +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import java.util.ArrayList; +import java.util.List; + +public class ClickhouseTestDataComparator extends AdvancedTestDataComparator { + + private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); + + @Override + protected List resolveIdentifier(final String identifier) { + final List result = new ArrayList<>(); + final String resolved = namingResolver.getIdentifier(identifier); + result.add(identifier); + result.add(resolved); + if (!resolved.startsWith("\"")) { + result.add(resolved.toLowerCase()); + result.add(resolved.toUpperCase()); + } + return result; + } + +} diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile index ca9d01ecea13..a6f118f4b63d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile @@ -2,5 +2,7 @@ FROM clickhouse/clickhouse-server:22.5 EXPOSE 8123 EXPOSE 8443 +EXPOSE 9000 +EXPOSE 9440 COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 077b3497e1ba..b8e9dcaa992e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.factory.DataSourceFactory; @@ -18,6 +19,7 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; +import java.util.Collections; import java.util.List; import java.util.Map; import javax.sql.DataSource; @@ -37,9 +39,11 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De private static final String PASSWORD = "password"; - static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( - "socket_timeout", "3000000", - "sslmode", "none"); + static final List SSL_PARAMETERS = ImmutableList.of( + "socket_timeout=3000000", + "sslmode=none"); + static final List DEFAULT_PARAMETERS = ImmutableList.of( + "socket_timeout=3000000"); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new ClickhouseDestination(), HOST_KEY, PORT_KEY); @@ -52,11 +56,18 @@ public ClickhouseDestination() { @Override public JsonNode toJdbcConfig(final JsonNode config) { final boolean isSsl = JdbcUtils.useSsl(config); - final String jdbcUrl = String.format("jdbc:clickhouse:%s://%s:%s/%s", + final StringBuilder jdbcUrl = new StringBuilder( + String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()); + config.get("port").asInt(), + config.get("database").asText())); + + if (isSsl) { + jdbcUrl.append("?").append(String.join("&", SSL_PARAMETERS)); + } else { + jdbcUrl.append("?").append(String.join("&", DEFAULT_PARAMETERS)); + } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) @@ -94,12 +105,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - if (JdbcUtils.useSsl(config)) { - return SSL_JDBC_PARAMETERS; - } else { - // No need for any parameters if the connection doesn't use SSL except socket_timeout - return ImmutableMap.of("socket_timeout", "3000000"); - } + return Collections.emptyMap(); } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index c288edc0cfdd..d48833fb6712 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -11,6 +11,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -91,20 +92,6 @@ static void cleanUp() { db.close(); } - @Test - void testDefaultParamsNoSSL() { - final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_NO_SSL)); - assertEquals(ImmutableMap.of("socket_timeout", "3000000"), defaultProperties); - } - - @Test - void testDefaultParamsWithSSL() { - final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_WITH_SSL)); - assertEquals(ClickhouseDestination.SSL_JDBC_PARAMETERS, defaultProperties); - } - @Test void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); @@ -131,7 +118,7 @@ void sanityTest() throws Exception { config.get("username").asText(), config.get("password").asText(), ClickhouseDestination.DRIVER_CLASS, - String.format("jdbc:clickhouse:%s://%s:%s/%s", + String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), ClickhouseDestination.HTTP_PROTOCOL, config.get("host").asText(), config.get("port").asText(), diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile index 856204a987cb..a6f118f4b63d 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/docker/Dockerfile @@ -2,7 +2,7 @@ FROM clickhouse/clickhouse-server:22.5 EXPOSE 8123 EXPOSE 8443 - -RUN apt-get update && apt-get install -y net-tools curl +EXPOSE 9000 +EXPOSE 9440 COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ From 484a8822c8a2e11449c6f465aadb2b2d049a6c3d Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 22 Jul 2022 14:24:28 +0700 Subject: [PATCH 07/15] 13539 Updated SSL for source-clickhouse-strict-encrypt --- ...StrictEncryptJdbcSourceAcceptanceTest.java | 36 +++++++++++++++---- .../resources/docker/Dockerfile | 8 +++++ .../resources/docker/clickhouse_certs.sh | 12 +++++++ .../test-integration/resources/ssl_ports.xml | 26 ++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh create mode 100644 airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 44d641cc1266..f5ba2ecddd64 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.JsonNode; @@ -13,6 +14,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Source; @@ -21,18 +23,30 @@ import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.ConnectorSpecification; import java.sql.JDBCType; +import java.time.Duration; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + public static final Integer HTTP_PORT = 8123; + public static final Integer NATIVE_PORT = 9000; + public static final Integer HTTPS_PORT = 8443; + public static final Integer NATIVE_SECURE_PORT = 9440; + private static final String DEFAULT_DB_NAME = "default"; + private static final String DEFAULT_USER_NAME = "default"; + private static GenericContainer container; private static JdbcDatabase db; private JsonNode config; @@ -67,16 +81,24 @@ public String createTableQuery(final String tableName, @BeforeAll static void init() { - container = new GenericContainer("etsybaev/clickhouse-with-ssl:dev").withExposedPorts(8443); + container = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") + .withFileFromClasspath("Dockerfile", "docker/Dockerfile") + .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) + .withEnv("TZ", "UTC") + .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); container.start(); } @BeforeEach public void setup() throws Exception { final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) - .put("username", "default") + .put("host", HostPortResolver.resolveHost(container)) + .put("port", HTTPS_PORT) + .put("username", DEFAULT_USER_NAME) + .put("database", DEFAULT_DB_NAME) .put("password", "") .build()); @@ -85,9 +107,11 @@ public void setup() throws Exception { configWithoutDbName.get("username").asText(), configWithoutDbName.get("password").asText(), ClickHouseSource.DRIVER_CLASS, - String.format("jdbc:clickhouse://%s:%s?ssl=true&sslmode=none", + String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString() + "?sslmode=none", + ClickHouseSource.HTTPS_PROTOCOL, configWithoutDbName.get("host").asText(), - configWithoutDbName.get("port").asText()))); + configWithoutDbName.get("port").asInt(), + configWithoutDbName.get("database").asText()))); dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile new file mode 100644 index 000000000000..a6f118f4b63d --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM clickhouse/clickhouse-server:22.5 + +EXPOSE 8123 +EXPOSE 8443 +EXPOSE 9000 +EXPOSE 9440 + +COPY clickhouse_certs.sh /docker-entrypoint-initdb.d/ diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh new file mode 100644 index 000000000000..285a2bc58c54 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/docker/clickhouse_certs.sh @@ -0,0 +1,12 @@ +echo "Preparing certs" + +openssl req -subj "/CN=my.host.name" -new \ + -newkey rsa:2048 -days 365 -nodes -x509 \ + -keyout /etc/clickhouse-server/server.key \ + -out /etc/clickhouse-server/server.crt + +openssl dhparam -out /etc/clickhouse-server/dhparam.pem 1024 + +chown $(id -u clickhouse):$(id -g clickhouse) /etc/clickhouse-server/server.{key,crt} + +echo "Finished preparing certs" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml new file mode 100644 index 000000000000..82018f7eae6e --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/ssl_ports.xml @@ -0,0 +1,26 @@ + + + 8123 + 8443 + 9000 + 9440 + 9004 + 9005 + 9009 + 9100 + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + relaxed + + ConsoleCertificateHandler + + false + true + sslv2,sslv3 + true + + + \ No newline at end of file From e7ef45d88b8996a4aca6fe63f7d9a56ee4c06c57 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 22 Jul 2022 15:01:20 +0700 Subject: [PATCH 08/15] 13539 Clean up --- .../sources/ClickHouseSourceAcceptanceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index df7fad4c3dfe..c4879c8d6cda 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -27,9 +27,11 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; +import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.wait.strategy.Wait; public class ClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { From 88d7c6e488def6445d5d38d20d4dcba49065fc45 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 22 Jul 2022 16:23:45 +0700 Subject: [PATCH 09/15] 13539 Fixed failed tests --- ...ClickhouseDestinationStrictEncryptAcceptanceTest.java | 9 ++++++++- .../clickhouse/ClickhouseDestinationTest.java | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 963625dafdf7..560478c19de3 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -24,6 +24,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; @@ -113,7 +114,13 @@ protected JsonNode getConfig() { // dbt clickhouse adapter uses native protocol, its default port is 9000 // Since we disabled normalization and dbt test, we only use the JDBC port here. return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull( + db.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, USER_NAME) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 073800e977b7..535cae2d7f99 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -121,7 +121,7 @@ void sanityTest() throws Exception { String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), ClickhouseDestination.HTTP_PROTOCOL, config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()))); final List actualRecords = database.bufferedResultSetQuery( From ad1a677da3bd44de74fe1d0b7f30d30f33ff646b Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 22 Jul 2022 16:38:26 +0700 Subject: [PATCH 10/15] 13539 Resolved host by ip --- .../integrations/util/HostPortResolver.java | 21 +++++++++++++------ ...estinationStrictEncryptAcceptanceTest.java | 9 +------- ...StrictEncryptJdbcSourceAcceptanceTest.java | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java index 8dfc9fcef9e9..4cd69abffecd 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java @@ -10,12 +10,8 @@ public class HostPortResolver { public static String resolveHost(GenericContainer container) { - return System.getProperty("os.name").toLowerCase().startsWith("mac") ? Objects.requireNonNull(container.getContainerInfo() - .getNetworkSettings() - .getNetworks() - .entrySet().stream() - .findFirst() - .get().getValue().getIpAddress()) + return System.getProperty("os.name").toLowerCase().startsWith("mac") + ? getIpAddress(container) : container.getHost(); } @@ -24,4 +20,17 @@ public static int resolvePort(GenericContainer container) { : container.getFirstMappedPort(); } + public static String resolveIpAddress(GenericContainer container) { + return getIpAddress(container); + } + + private static String getIpAddress(GenericContainer container) { + return Objects.requireNonNull(container.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress()); + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 560478c19de3..26697187571a 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -24,7 +24,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; @@ -114,13 +113,7 @@ protected JsonNode getConfig() { // dbt clickhouse adapter uses native protocol, its default port is 9000 // Since we disabled normalization and dbt test, we only use the JDBC port here. return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull( - db.getContainerInfo() - .getNetworkSettings() - .getNetworks() - .entrySet().stream() - .findFirst() - .get().getValue().getIpAddress())) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db)) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, USER_NAME) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 20203b49ab37..2cfedd119a5b 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -96,7 +96,7 @@ static void init() { @BeforeEach public void setup() throws Exception { final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(container)) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.USERNAME_KEY, DEFAULT_USER_NAME) .put("database", DEFAULT_DB_NAME) From 630d8ae63a365fd56f37643e2b082e4800a7a5af Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Mon, 25 Jul 2022 15:25:18 +0700 Subject: [PATCH 11/15] 13539 Fixed code formatting --- ...houseDestinationStrictEncryptAcceptanceTest.java | 13 ++++++------- .../clickhouse/ClickhouseDestination.java | 8 ++++---- .../clickhouse/ClickhouseSqlOperations.java | 2 +- ...kHouseStrictEncryptJdbcSourceAcceptanceTest.java | 10 +++++----- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 26697187571a..c275ca914410 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -173,11 +173,11 @@ protected void setup(final TestDestinationEnv testEnv) { db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") .withFileFromClasspath("Dockerfile", "docker/Dockerfile") .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) - .withEnv("TZ", "UTC") - .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) - .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) - .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + .withEnv("TZ", "UTC") + .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); db.start(); @@ -185,8 +185,7 @@ protected void setup(final TestDestinationEnv testEnv) { HTTP_PORT, db.getMappedPort(HTTP_PORT), HTTPS_PORT, db.getMappedPort(HTTPS_PORT), NATIVE_PORT, db.getMappedPort(NATIVE_PORT), - NATIVE_SECURE_PORT, db.getMappedPort(NATIVE_SECURE_PORT) - )); + NATIVE_SECURE_PORT, db.getMappedPort(NATIVE_SECURE_PORT))); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 71cfd7ffbabf..e2d8ac9152d8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -58,10 +58,10 @@ public JsonNode toJdbcConfig(final JsonNode config) { final boolean isSsl = JdbcUtils.useSsl(config); final StringBuilder jdbcUrl = new StringBuilder( String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), - isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText())); + isSsl ? HTTPS_PROTOCOL : HTTP_PROTOCOL, + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), + config.get(JdbcUtils.DATABASE_KEY).asText())); if (isSsl) { jdbcUrl.append("?").append(String.join("&", SSL_PARAMETERS)); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index fb182e249e3a..3b9e7921e51c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.clickhouse; +import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.ClickHouseStatement; -import com.clickhouse.client.ClickHouseFormat; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 2cfedd119a5b..7a409be9ad20 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -85,11 +85,11 @@ static void init() { container = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") .withFileFromClasspath("Dockerfile", "docker/Dockerfile") .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) - .withEnv("TZ", "UTC") - .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) - .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) - .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + .withEnv("TZ", "UTC") + .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); container.start(); } From d79ccfde99e8e49447281ac19156f1fd1a8c6af0 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Tue, 26 Jul 2022 11:29:45 +0700 Subject: [PATCH 12/15] 13539 Bump up source-clickhouse-strict-encrypt version --- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 8555fba96344..09e31f5ecfab 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt From a2be5dbd43641b5159a094651a9fe1fcee94c150 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Tue, 26 Jul 2022 11:41:15 +0700 Subject: [PATCH 13/15] 13539 Bump up source-clickhouse-strict-encrypt version --- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 09e31f5ecfab..14403b9faab8 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt From b39df161bdd8303f60fd44fa3b3c69bf7084854c Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 26 Jul 2022 04:58:15 +0000 Subject: [PATCH 14/15] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 057466db607c..ccbeb4ee1e80 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -152,7 +152,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse icon: cliskhouse.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9e7226eff190..91587c2df13e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1179,7 +1179,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.10" +- dockerImage: "airbyte/source-clickhouse:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -1191,7 +1191,6 @@ - "port" - "database" - "username" - additionalProperties: false properties: host: description: "The host endpoint of the Clickhouse cluster." From b10fe396da3995be8d0b537732a12721cc1740a4 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 26 Jul 2022 06:03:53 +0000 Subject: [PATCH 15/15] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e5cc45733cab..b62642213b3f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Databricks Lakehouse diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index bed1cd7d1bb5..458d3511c8c1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -797,7 +797,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.10" +- dockerImage: "airbyte/destination-clickhouse:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: