13539 Fix integration tests source-clickhouse Mac OS

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -67,7 +67,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
   releaseStage: alpha
 - name: Databricks Lakehouse

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

@@ -797,7 +797,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.1.10"
+- dockerImage: "airbyte/destination-clickhouse:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -152,7 +152,7 @@
 - name: ClickHouse
   sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c
   dockerRepository: airbyte/source-clickhouse
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse
   icon: cliskhouse.svg
   sourceType: database

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -1179,7 +1179,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-clickhouse:0.1.10"
+- dockerImage: "airbyte/source-clickhouse:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -1191,7 +1191,6 @@
       - "port"
       - "database"
       - "username"
-      additionalProperties: false
       properties:
         host:
           description: "The host endpoint of the Clickhouse cluster."

airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java

@@ -9,7 +9,7 @@
  */
 public enum DatabaseDriver {
 
-  CLICKHOUSE("com.clickhouse.jdbc.ClickHouseDriver", "jdbc:clickhouse://%s:%d/%s"),
+  CLICKHOUSE("com.clickhouse.jdbc.ClickHouseDriver", "jdbc:clickhouse:%s://%s:%d/%s"),
   DATABRICKS("com.databricks.client.jdbc.Driver", "jdbc:databricks://%s;HttpPath=%s;UserAgentEntry=Airbyte"),
   DB2("com.ibm.db2.jcc.DB2Driver", "jdbc:db2://%s:%d/%s"),
   MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"),

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java

@@ -10,12 +10,8 @@
 public class HostPortResolver {
 
   public static String resolveHost(GenericContainer container) {
-    return System.getProperty("os.name").toLowerCase().startsWith("mac") ? Objects.requireNonNull(container.getContainerInfo()
-        .getNetworkSettings()
-        .getNetworks()
-        .entrySet().stream()
-        .findFirst()
-        .get().getValue().getIpAddress())
+    return System.getProperty("os.name").toLowerCase().startsWith("mac")
+        ? getIpAddress(container)
         : container.getHost();
   }
 
@@ -24,4 +20,17 @@ public static int resolvePort(GenericContainer container) {
         : container.getFirstMappedPort();
   }
 
+  public static String resolveIpAddress(GenericContainer container) {
+    return getIpAddress(container);
+  }
+
+  private static String getIpAddress(GenericContainer container) {
+    return Objects.requireNonNull(container.getContainerInfo()
+        .getNetworkSettings()
+        .getNetworks()
+        .entrySet().stream()
+        .findFirst()
+        .get().getValue().getIpAddress());
+  }
+
 }

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle

@@ -17,8 +17,7 @@ dependencies {
     implementation project(':airbyte-integrations:connectors:destination-clickhouse')
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 
-    // https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc
-    implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1-patch'
+    implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all'
 
     // https://mvnrepository.com/artifact/org.testcontainers/clickhouse
     testImplementation libs.connectors.destination.testcontainers.clickhouse

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java

@@ -4,41 +4,60 @@
 
 package io.airbyte.integrations.destination.clickhouse;
 
+import static java.time.temporal.ChronoUnit.SECONDS;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.db.factory.DataSourceFactory;
+import io.airbyte.db.factory.DatabaseDriver;
 import io.airbyte.db.jdbc.DefaultJdbcDatabase;
 import io.airbyte.db.jdbc.JdbcDatabase;
 import io.airbyte.db.jdbc.JdbcUtils;
 import io.airbyte.integrations.base.JavaBaseConstants;
 import io.airbyte.integrations.destination.ExtendedNameTransformer;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
+import io.airbyte.integrations.util.HostPortResolver;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Disabled;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.BindMode;
-import org.testcontainers.containers.ClickHouseContainer;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.images.builder.ImageFromDockerfile;
 
 public class ClickhouseDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ClickhouseDestinationStrictEncryptAcceptanceTest.class);
 
-  private static final String DB_NAME = "default";
-
-  private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
-
-  private ClickHouseContainer db;
-
   public static final Integer HTTP_PORT = 8123;
   public static final Integer NATIVE_PORT = 9000;
   public static final Integer HTTPS_PORT = 8443;
   public static final Integer NATIVE_SECURE_PORT = 9440;
+  private static final String DB_NAME = "default";
+  private static final String USER_NAME = "default";
+  private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
+  private GenericContainer db;
+
+  private static JdbcDatabase getDatabase(final JsonNode config) {
+    final String jdbcStr = String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString() + "?sslmode=none",
+        ClickhouseDestination.HTTPS_PROTOCOL,
+        config.get(JdbcUtils.HOST_KEY).asText(),
+        config.get(JdbcUtils.PORT_KEY).asInt(),
+        config.get(JdbcUtils.DATABASE_KEY).asText());
+    return new DefaultJdbcDatabase(DataSourceFactory.create(
+        config.get(JdbcUtils.USERNAME_KEY).asText(),
+        config.has(JdbcUtils.PASSWORD_KEY) ? config.get(JdbcUtils.PASSWORD_KEY).asText() : null,
+        ClickhouseDestination.DRIVER_CLASS,
+        jdbcStr));
+  }
 
   @Override
   protected String getImageName() {
@@ -47,7 +66,7 @@ protected String getImageName() {
 
   @Override
   protected boolean supportsNormalization() {
-    return true;
+    return false;
   }
 
   @Override
@@ -60,6 +79,26 @@ protected boolean implementsNamespaces() {
     return true;
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new ClickhouseTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected String getDefaultSchema(final JsonNode config) {
     if (config.get(JdbcUtils.DATABASE_KEY) == null) {
@@ -74,11 +113,11 @@ protected JsonNode getConfig() {
     // dbt clickhouse adapter uses native protocol, its default port is 9000
     // Since we disabled normalization and dbt test, we only use the JDBC port here.
     return Jsons.jsonNode(ImmutableMap.builder()
-        .put(JdbcUtils.HOST_KEY, db.getHost())
-        .put(JdbcUtils.PORT_KEY, db.getMappedPort(HTTPS_PORT))
+        .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db))
+        .put(JdbcUtils.PORT_KEY, HTTPS_PORT)
         .put(JdbcUtils.DATABASE_KEY, DB_NAME)
-        .put(JdbcUtils.USERNAME_KEY, db.getUsername())
-        .put(JdbcUtils.PASSWORD_KEY, db.getPassword())
+        .put(JdbcUtils.USERNAME_KEY, USER_NAME)
+        .put(JdbcUtils.PASSWORD_KEY, "")
         .put(JdbcUtils.SCHEMA_KEY, DB_NAME)
         .build());
   }
@@ -129,31 +168,24 @@ protected List<String> resolveIdentifier(final String identifier) {
     return result;
   }
 
-  private static JdbcDatabase getDatabase(final JsonNode config) {
-    final String jdbcStr = String.format("jdbc:clickhouse://%s:%s/%s?ssl=true&sslmode=none",
-        config.get(JdbcUtils.HOST_KEY).asText(),
-        config.get(JdbcUtils.PORT_KEY).asText(),
-        config.get(JdbcUtils.DATABASE_KEY).asText());
-    return new DefaultJdbcDatabase(DataSourceFactory.create(
-        config.get(JdbcUtils.USERNAME_KEY).asText(),
-        config.has(JdbcUtils.PASSWORD_KEY) ? config.get(JdbcUtils.PASSWORD_KEY).asText() : null,
-        ClickhouseDestination.DRIVER_CLASS,
-        jdbcStr));
-  }
-
   @Override
   protected void setup(final TestDestinationEnv testEnv) {
-    db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server")
-        .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT)
-        .withClasspathResourceMapping("config.xml", "/etc/clickhouse-server/config.xml", BindMode.READ_ONLY)
-        .withClasspathResourceMapping("server.crt", "/etc/clickhouse-server/server.crt", BindMode.READ_ONLY)
-        .withClasspathResourceMapping("server.key", "/etc/clickhouse-server/server.key", BindMode.READ_ONLY)
-        .withClasspathResourceMapping("dhparam.pem", "/etc/clickhouse-server/dhparam.pem", BindMode.READ_ONLY);
+    db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test")
+        .withFileFromClasspath("Dockerfile", "docker/Dockerfile")
+        .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh"))
+            .withEnv("TZ", "UTC")
+            .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT)
+            .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY)
+            .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT)
+                .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS)));
+
     db.start();
 
-    LOGGER.info(String.format("Clickhouse server container port mapping: %d -> %d, %d -> %d",
+    LOGGER.info(String.format("Clickhouse server container port mapping: %d -> %d, %d -> %d, %d -> %d, %d -> %d",
         HTTP_PORT, db.getMappedPort(HTTP_PORT),
-        HTTPS_PORT, db.getMappedPort(HTTPS_PORT)));
+        HTTPS_PORT, db.getMappedPort(HTTPS_PORT),
+        NATIVE_PORT, db.getMappedPort(NATIVE_PORT),
+        NATIVE_SECURE_PORT, db.getMappedPort(NATIVE_SECURE_PORT)));
   }
 
   @Override
@@ -202,4 +234,9 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin
     super.testSyncWithNormalization(messagesFilename, catalogFilename);
   }
 
+  @Disabled
+  public void specNormalizationValueShouldBeCorrect() throws Exception {
+    super.specNormalizationValueShouldBeCorrect();
+  }
+
 }

airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.clickhouse;
+
+import io.airbyte.integrations.destination.ExtendedNameTransformer;
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ClickhouseTestDataComparator extends AdvancedTestDataComparator {
+
+  private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
+
+  @Override
+  protected List<String> resolveIdentifier(final String identifier) {
+    final List<String> result = new ArrayList<>();
+    final String resolved = namingResolver.getIdentifier(identifier);
+    result.add(identifier);
+    result.add(resolved);
+    if (!resolved.startsWith("\"")) {
+      result.add(resolved.toLowerCase());
+      result.add(resolved.toUpperCase());
+    }
+    return result;
+  }
+
+}