Greg/clickhouse polishing (airbytehq#17483)

* add icon for clickhouse in destination folder * use http port only in clickhouse * declare driver: http for dbt explicitly * bump destination clickhouse version Co-authored-by: restrry <restrry@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
jhammarstedt · Oct 31, 2022 · 79dd61c · 79dd61c
1 parent 07ba4a0
commit 79dd61c
Show file tree

Hide file tree

Showing 18 changed files with 32 additions and 95 deletions.
diff --git a/...t/src/main/resources/icons/cliskhouse.svg → ...t/src/main/resources/icons/clickhouse.svg b/...t/src/main/resources/icons/cliskhouse.svg → ...t/src/main/resources/icons/clickhouse.svg
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -67,7 +67,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.1.12
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
   releaseStage: alpha
 - name: Cloudflare R2

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -797,7 +797,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.1.12"
+- dockerImage: "airbyte/destination-clickhouse:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -818,53 +818,43 @@
           order: 0
         port:
           title: "Port"
-          description: "JDBC port (not the native port) of the database."
+          description: "HTTP port of the database."
           type: "integer"
           minimum: 0
           maximum: 65536
           default: 8123
           examples:
           - "8123"
           order: 1
-        tcp-port:
-          title: "Native Port"
-          description: "Native port (not the JDBC) of the database."
-          type: "integer"
-          minimum: 0
-          maximum: 65536
-          default: 9000
-          examples:
-          - "9000"
-          order: 2
         database:
           title: "DB Name"
           description: "Name of the database."
           type: "string"
-          order: 3
+          order: 2
         username:
           title: "User"
           description: "Username to use to access the database."
           type: "string"
-          order: 4
+          order: 3
         password:
           title: "Password"
           description: "Password associated with the username."
           type: "string"
           airbyte_secret: true
-          order: 5
+          order: 4
         jdbc_url_params:
           description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
             \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
           title: "JDBC URL Params"
           type: "string"
-          order: 6
+          order: 5
         ssl:
           title: "SSL Connection"
           description: "Encrypt data using SSL."
           type: "boolean"
           default: false
-          order: 7
+          order: 6
         tunnel_method:
           type: "object"
           title: "SSH Tunnel Method"

diff --git a/airbyte-config/init/src/test/resources/connector_catalog.json b/airbyte-config/init/src/test/resources/connector_catalog.json
@@ -791,24 +791,14 @@
             },
             "port": {
               "title": "Port",
-              "description": "JDBC port (not the native port) of the database.",
+              "description": "HTTP port of the database.",
               "type": "integer",
               "minimum": 0,
               "maximum": 65536,
               "default": 8123,
               "examples": ["8123"],
               "order": 1
             },
-            "tcp-port": {
-              "title": "Native Port",
-              "description": "Native port (not the JDBC) of the database.",
-              "type": "integer",
-              "minimum": 0,
-              "maximum": 65536,
-              "default": 9000,
-              "examples": ["9000"],
-              "order": 2
-            },
             "database": {
               "title": "DB Name",
               "description": "Name of the database.",
@@ -6120,7 +6110,7 @@
       "dockerRepository": "airbyte/source-clickhouse-strict-encrypt",
       "dockerImageTag": "0.1.8",
       "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse",
-      "icon": "cliskhouse.svg",
+      "icon": "clickhouse.svg",
       "sourceType": "database",
       "spec": {
         "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse",

diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java
@@ -41,7 +41,7 @@ public class JdbcUtils {
   public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
   public static final String PASSWORD_KEY = "password";
   public static final String PORT_KEY = "port";
-  public static final String TCP_PORT_KEY = "tcp-port";
+
   public static final List<String> PORT_LIST_KEY = List.of("port");
   public static final String SCHEMA_KEY = "schema";
   // NOTE: this is the plural version of SCHEMA_KEY

diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py
@@ -27,7 +27,6 @@
 NORMALIZATION_TEST_MYSQL_DB_PORT = "NORMALIZATION_TEST_MYSQL_DB_PORT"
 NORMALIZATION_TEST_POSTGRES_DB_PORT = "NORMALIZATION_TEST_POSTGRES_DB_PORT"
 NORMALIZATION_TEST_CLICKHOUSE_DB_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_PORT"
-NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT"
 NORMALIZATION_TEST_TIDB_DB_PORT = "NORMALIZATION_TEST_TIDB_DB_PORT"
 
 
@@ -224,28 +223,20 @@ def setup_mssql_db(self):
 
     def setup_clickhouse_db(self):
         """
-        ClickHouse official JDBC driver use HTTP port 8123, while Python ClickHouse
-        driver uses native port 9000, so we need to open both ports for destination
-        connector and dbt container respectively.
+        ClickHouse official JDBC driver uses HTTP port 8123.
 
         Ref: https://altinity.com/blog/2019/3/15/clickhouse-networking-part-1
         """
         start_db = True
         port = 8123
-        tcp_port = 9000
         if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT):
             port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT))
             start_db = False
-        if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT):
-            tcp_port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT))
-            start_db = False
         if start_db:
             port = self.find_free_port()
-            tcp_port = self.find_free_port()
         config = {
             "host": "localhost",
             "port": port,
-            "tcp-port": tcp_port,
             "database": self.target_schema,
             "username": "default",
             "password": "",
@@ -263,8 +254,6 @@ def setup_clickhouse_db(self):
                 "--ulimit",
                 "nofile=262144:262144",
                 "-p",
-                f"{config['tcp-port']}:9000",  # Python clickhouse driver use native port
-                "-p",
                 f"{config['port']}:8123",  # clickhouse JDBC driver use HTTP port
                 "-d",
                 # so far, only the latest version ClickHouse server image turned on

diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py
@@ -318,7 +318,8 @@ def transform_clickhouse(config: Dict[str, Any]):
         # https://docs.getdbt.com/reference/warehouse-profiles/clickhouse-profile
         dbt_config = {
             "type": "clickhouse",
-            "driver": "native",
+            "driver": "http",
+            "verify": False,
             "host": config["host"],
             "port": config["port"],
             "schema": config["database"],
@@ -327,8 +328,6 @@ def transform_clickhouse(config: Dict[str, Any]):
         }
         if "password" in config:
             dbt_config["password"] = config["password"]
-        if "tcp-port" in config:
-            dbt_config["port"] = config["tcp-port"]
         return dbt_config
 
     @staticmethod

diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py
@@ -440,7 +440,8 @@ def test_transform_clickhouse(self):
         actual = TransformConfig().transform_clickhouse(input)
         expected = {
             "type": "clickhouse",
-            "driver": "native",
+            "driver": "http",
+            "verify": False,
             "host": "airbyte.io",
             "port": 9440,
             "schema": "default",

diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt
diff --git a/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java
@@ -113,7 +113,6 @@ protected String getDefaultSchema(final JsonNode config) {
   protected JsonNode getConfig() {
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db))
-        .put(JdbcUtils.TCP_PORT_KEY, NATIVE_SECURE_PORT)
         .put(JdbcUtils.PORT_KEY, HTTPS_PORT)
         .put(JdbcUtils.DATABASE_KEY, DB_NAME)
         .put(JdbcUtils.USERNAME_KEY, USER_NAME)

diff --git a/...ns/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/...ns/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json
@@ -19,48 +19,38 @@
       },
       "port": {
         "title": "Port",
-        "description": "JDBC port (not the native port) of the database.",
+        "description": "HTTP port of the database.",
         "type": "integer",
         "minimum": 0,
         "maximum": 65536,
         "default": 8123,
         "examples": ["8123"],
         "order": 1
       },
-      "tcp-port": {
-        "title": "Native Port",
-        "description": "Native port (not the JDBC) of the database.",
-        "type": "integer",
-        "minimum": 0,
-        "maximum": 65536,
-        "default": 9000,
-        "examples": ["9000"],
-        "order": 2
-      },
       "database": {
         "title": "DB Name",
         "description": "Name of the database.",
         "type": "string",
-        "order": 3
+        "order": 2
       },
       "username": {
         "title": "User",
         "description": "Username to use to access the database.",
         "type": "string",
-        "order": 4
+        "order": 3
       },
       "password": {
         "title": "Password",
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true,
-        "order": 5
+        "order": 4
       },
       "jdbc_url_params": {
         "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
         "title": "JDBC URL Params",
         "type": "string",
-        "order": 6
+        "order": 5
       },
       "tunnel_method": {
         "type": "object",

diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/destination-clickhouse
diff --git a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md
@@ -10,12 +10,8 @@ This destination connector uses ClickHouse official JDBC driver, which uses HTTP
 
 ## Quick Notes
 
-- ClickHouse JDBC driver uses HTTP protocal (default 8123) but [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) use TCP protocal (default 9000).
-
 - This connector doesn't support nested streams and schema change yet.
 
-- The community [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) has some bugs haven't been fixed yet, for example [https://github.com/silentsokolov/dbt-clickhouse/issues/20](https://github.com/silentsokolov/dbt-clickhouse/issues/20), so the dbt test is based on a fork [https://github.com/burmecia/dbt-clickhouse](https://github.com/burmecia/dbt-clickhouse).
-
 ## API Reference
 
 The ClickHouse reference documents: [https://clickhouse.com/docs/en/](https://clickhouse.com/docs/en/)

diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json
@@ -19,55 +19,45 @@
       },
       "port": {
         "title": "Port",
-        "description": "JDBC port (not the native port) of the database.",
+        "description": "HTTP port of the database.",
         "type": "integer",
         "minimum": 0,
         "maximum": 65536,
         "default": 8123,
         "examples": ["8123"],
         "order": 1
       },
-      "tcp-port": {
-        "title": "Native Port",
-        "description": "Native port (not the JDBC) of the database.",
-        "type": "integer",
-        "minimum": 0,
-        "maximum": 65536,
-        "default": 9000,
-        "examples": ["9000"],
-        "order": 2
-      },
       "database": {
         "title": "DB Name",
         "description": "Name of the database.",
         "type": "string",
-        "order": 3
+        "order": 2
       },
       "username": {
         "title": "User",
         "description": "Username to use to access the database.",
         "type": "string",
-        "order": 4
+        "order": 3
       },
       "password": {
         "title": "Password",
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true,
-        "order": 5
+        "order": 4
       },
       "jdbc_url_params": {
         "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
         "title": "JDBC URL Params",
         "type": "string",
-        "order": 6
+        "order": 5
       },
       "ssl": {
         "title": "SSL Connection",
         "description": "Encrypt data using SSL.",
         "type": "boolean",
         "default": false,
-        "order": 7
+        "order": 6
       }
     }
   }

diff --git a/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java
@@ -89,15 +89,9 @@ protected String getDefaultSchema(final JsonNode config) {
 
   @Override
   protected JsonNode getConfig() {
-    final Optional tcpPort = db.getExposedPorts().stream()
-        .map(exPort -> db.getMappedPort((Integer) exPort))
-        .filter(el -> !db.getFirstMappedPort().equals(el))
-        .findFirst();
-
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db))
         .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db))
-        .put(JdbcUtils.TCP_PORT_KEY, tcpPort.get())
         .put(JdbcUtils.DATABASE_KEY, DB_NAME)
         .put(JdbcUtils.USERNAME_KEY, db.getUsername())
         .put(JdbcUtils.PASSWORD_KEY, db.getPassword())

diff --git a/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java
@@ -94,7 +94,7 @@ protected String getDefaultSchema(final JsonNode config) {
   @Override
   protected JsonNode getConfig() throws Exception {
     return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME)
-        .put("schema", DB_NAME).put(JdbcUtils.TCP_PORT_KEY, 9000));
+        .put("schema", DB_NAME));
   }
 
   @Override

diff --git a/...st/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java b/...st/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java
@@ -34,7 +34,6 @@ public class ClickhouseDestinationSpecTest {
       + "\"username\" : \"clickhouse\",  "
       + "\"database\" : \"clickhouse_db\",  "
       + "\"port\" : 8123,  "
-      + "\"tcp-port\" : 9000,  "
       + "\"host\" : \"localhost\",  "
       + "\"jdbc_url_params\" : \"property1=pValue1&property2=pValue2\",  "
       + "\"ssl\" : true "

diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md
@@ -58,8 +58,7 @@ You will need to choose an existing database or create a new database that will
 You should now have all the requirements needed to configure ClickHouse as a destination in the UI. You'll need the following information to configure the ClickHouse destination:
 
 * **Host**
-* **Port** (JDBC HTTP port, not the native port)
-* **Tcp-port** (Native port, also required for data normalization)
+* **Port**
 * **Username**
 * **Password**
 * **Database**
@@ -81,6 +80,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t
 
 | Version | Date       | Pull Request | Subject                                      |
 |:--------|:-----------| :--- |:---------------------------------------------|
+| 0.2.0  | 2022-09-27 | [16970](https://github.com/airbytehq/airbyte/pull/16970)   | Remove TCP port from spec parameters |
 | 0.1.12  | 2022-09-08 | [16444](https://github.com/airbytehq/airbyte/pull/16444)   | Added custom jdbc params field |
 | 0.1.10  | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9 |
 | 0.1.8   | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout |