diff --git a/airbyte-config/init/src/main/resources/icons/cliskhouse.svg b/airbyte-config/init/src/main/resources/icons/clickhouse.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/cliskhouse.svg rename to airbyte-config/init/src/main/resources/icons/clickhouse.svg diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 10cb3ec2c6ab..1b25393b2108 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.12 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Cloudflare R2 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 050f9bb10ca5..a27d3317222a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -797,7 +797,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.12" +- dockerImage: "airbyte/destination-clickhouse:0.2.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -818,7 +818,7 @@ order: 0 port: title: "Port" - description: "JDBC port (not the native port) of the database." + description: "HTTP port of the database." type: "integer" minimum: 0 maximum: 65536 @@ -826,45 +826,35 @@ examples: - "8123" order: 1 - tcp-port: - title: "Native Port" - description: "Native port (not the JDBC) of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 9000 - examples: - - "9000" - order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 3 + order: 2 username: title: "User" description: "Username to use to access the database." type: "string" - order: 4 + order: 3 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 5 + order: 4 jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" type: "string" - order: 6 + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 7 + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-config/init/src/test/resources/connector_catalog.json b/airbyte-config/init/src/test/resources/connector_catalog.json index a8da005c1479..c65a2d07cc35 100644 --- a/airbyte-config/init/src/test/resources/connector_catalog.json +++ b/airbyte-config/init/src/test/resources/connector_catalog.json @@ -791,7 +791,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -799,16 +799,6 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", @@ -6120,7 +6110,7 @@ "dockerRepository": "airbyte/source-clickhouse-strict-encrypt", "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse", - "icon": "cliskhouse.svg", + "icon": "clickhouse.svg", "sourceType": "database", "spec": { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 2e1aea25d215..e359a2331889 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -41,7 +41,7 @@ public class JdbcUtils { public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; public static final String PASSWORD_KEY = "password"; public static final String PORT_KEY = "port"; - public static final String TCP_PORT_KEY = "tcp-port"; + public static final List PORT_LIST_KEY = List.of("port"); public static final String SCHEMA_KEY = "schema"; // NOTE: this is the plural version of SCHEMA_KEY diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index 43d50c67daa4..80726e45e779 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -27,7 +27,6 @@ NORMALIZATION_TEST_MYSQL_DB_PORT = "NORMALIZATION_TEST_MYSQL_DB_PORT" NORMALIZATION_TEST_POSTGRES_DB_PORT = "NORMALIZATION_TEST_POSTGRES_DB_PORT" NORMALIZATION_TEST_CLICKHOUSE_DB_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_PORT" -NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT" NORMALIZATION_TEST_TIDB_DB_PORT = "NORMALIZATION_TEST_TIDB_DB_PORT" @@ -224,28 +223,20 @@ def setup_mssql_db(self): def setup_clickhouse_db(self): """ - ClickHouse official JDBC driver use HTTP port 8123, while Python ClickHouse - driver uses native port 9000, so we need to open both ports for destination - connector and dbt container respectively. + ClickHouse official JDBC driver uses HTTP port 8123. Ref: https://altinity.com/blog/2019/3/15/clickhouse-networking-part-1 """ start_db = True port = 8123 - tcp_port = 9000 if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT): port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT)) start_db = False - if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT): - tcp_port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT)) - start_db = False if start_db: port = self.find_free_port() - tcp_port = self.find_free_port() config = { "host": "localhost", "port": port, - "tcp-port": tcp_port, "database": self.target_schema, "username": "default", "password": "", @@ -263,8 +254,6 @@ def setup_clickhouse_db(self): "--ulimit", "nofile=262144:262144", "-p", - f"{config['tcp-port']}:9000", # Python clickhouse driver use native port - "-p", f"{config['port']}:8123", # clickhouse JDBC driver use HTTP port "-d", # so far, only the latest version ClickHouse server image turned on diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 756cdba77e9a..a762b39f1a45 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -318,7 +318,8 @@ def transform_clickhouse(config: Dict[str, Any]): # https://docs.getdbt.com/reference/warehouse-profiles/clickhouse-profile dbt_config = { "type": "clickhouse", - "driver": "native", + "driver": "http", + "verify": False, "host": config["host"], "port": config["port"], "schema": config["database"], @@ -327,8 +328,6 @@ def transform_clickhouse(config: Dict[str, Any]): } if "password" in config: dbt_config["password"] = config["password"] - if "tcp-port" in config: - dbt_config["port"] = config["tcp-port"] return dbt_config @staticmethod diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 337afaca4b12..2adbb2f441cf 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -440,7 +440,8 @@ def test_transform_clickhouse(self): actual = TransformConfig().transform_clickhouse(input) expected = { "type": "clickhouse", - "driver": "native", + "driver": "http", + "verify": False, "host": "airbyte.io", "port": 9440, "schema": "default", diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 50a0831ce3fe..8f27e4f6b615 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 0294dd1fecfb..741bded1ba21 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -113,7 +113,6 @@ protected String getDefaultSchema(final JsonNode config) { protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db)) - .put(JdbcUtils.TCP_PORT_KEY, NATIVE_SECURE_PORT) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, USER_NAME) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 2238b3870b70..0cc91dd63bf1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,40 +27,30 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "tunnel_method": { "type": "object", diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index f9eeeb4d8d7f..0f5a4cf4c350 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md index 13d96b595110..c728bde55a2e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md +++ b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md @@ -10,12 +10,8 @@ This destination connector uses ClickHouse official JDBC driver, which uses HTTP ## Quick Notes -- ClickHouse JDBC driver uses HTTP protocal (default 8123) but [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) use TCP protocal (default 9000). - - This connector doesn't support nested streams and schema change yet. -- The community [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) has some bugs haven't been fixed yet, for example [https://github.com/silentsokolov/dbt-clickhouse/issues/20](https://github.com/silentsokolov/dbt-clickhouse/issues/20), so the dbt test is based on a fork [https://github.com/burmecia/dbt-clickhouse](https://github.com/burmecia/dbt-clickhouse). - ## API Reference The ClickHouse reference documents: [https://clickhouse.com/docs/en/](https://clickhouse.com/docs/en/) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index adf44db3857b..94f5acffb296 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,47 +27,37 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", "default": false, - "order": 7 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index b2688949c7db..0dfd1db487fb 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -89,15 +89,9 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() { - final Optional tcpPort = db.getExposedPorts().stream() - .map(exPort -> db.getMappedPort((Integer) exPort)) - .filter(el -> !db.getFirstMappedPort().equals(el)) - .findFirst(); - return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.TCP_PORT_KEY, tcpPort.get()) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 9cb7aabac298..a76e5983f8e1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -94,7 +94,7 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() throws Exception { return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME) - .put("schema", DB_NAME).put(JdbcUtils.TCP_PORT_KEY, 9000)); + .put("schema", DB_NAME)); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java index e935eff44730..9577ff5db275 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java @@ -34,7 +34,6 @@ public class ClickhouseDestinationSpecTest { + "\"username\" : \"clickhouse\", " + "\"database\" : \"clickhouse_db\", " + "\"port\" : 8123, " - + "\"tcp-port\" : 9000, " + "\"host\" : \"localhost\", " + "\"jdbc_url_params\" : \"property1=pValue1&property2=pValue2\", " + "\"ssl\" : true " diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index bf7dd37df0b9..3ffb6ffb2459 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -58,8 +58,7 @@ You will need to choose an existing database or create a new database that will You should now have all the requirements needed to configure ClickHouse as a destination in the UI. You'll need the following information to configure the ClickHouse destination: * **Host** -* **Port** (JDBC HTTP port, not the native port) -* **Tcp-port** (Native port, also required for data normalization) +* **Port** * **Username** * **Password** * **Database** @@ -81,6 +80,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:---------------------------------------------| +| 0.2.0 | 2022-09-27 | [16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters | | 0.1.12 | 2022-09-08 | [16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field | | 0.1.10 | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9 | | 0.1.8 | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout |