From 32d32eafd44e25adbdef1f4ea46ba2fa1cdc34c0 Mon Sep 17 00:00:00 2001 From: restrry Date: Wed, 21 Sep 2022 13:33:42 +0200 Subject: [PATCH 1/9] add icon for clickhouse in destination folder --- .../init/src/main/resources/seed/destination_definitions.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 63a42fbd4853..4c73514611bd 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -70,6 +70,7 @@ dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha + icon: cliskhouse.svg - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks From bb47de8982840ca8c48032f977e70a58d8cbbfdf Mon Sep 17 00:00:00 2001 From: restrry Date: Wed, 21 Sep 2022 14:41:23 +0200 Subject: [PATCH 2/9] use http port only in clickhouse --- .../resources/seed/destination_specs.yaml | 22 +++++-------------- .../src/test/resources/connector_catalog.json | 12 +--------- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 2 +- .../integration_tests/dbt_integration_test.py | 13 +---------- .../transform_config/transform.py | 2 -- ...estinationStrictEncryptAcceptanceTest.java | 1 - .../src/test/resources/expected_spec.json | 20 +++++------------ .../destination-clickhouse/bootstrap.md | 4 ---- .../src/main/resources/spec.json | 22 +++++-------------- .../ClickhouseDestinationAcceptanceTest.java | 6 ----- ...shClickhouseDestinationAcceptanceTest.java | 2 +- .../ClickhouseDestinationSpecTest.java | 1 - docs/integrations/destinations/clickhouse.md | 3 +-- 13 files changed, 22 insertions(+), 88 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index a2a8b3842cc3..ce87894dce89 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -818,7 +818,7 @@ order: 0 port: title: "Port" - description: "JDBC port (not the native port) of the database." + description: "HTTP port of the database." type: "integer" minimum: 0 maximum: 65536 @@ -826,45 +826,35 @@ examples: - "8123" order: 1 - tcp-port: - title: "Native Port" - description: "Native port (not the JDBC) of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 9000 - examples: - - "9000" - order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 3 + order: 2 username: title: "User" description: "Username to use to access the database." type: "string" - order: 4 + order: 3 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 5 + order: 4 jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" type: "string" - order: 6 + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 7 + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-config/init/src/test/resources/connector_catalog.json b/airbyte-config/init/src/test/resources/connector_catalog.json index 3c70b3f12c63..29b15188923c 100644 --- a/airbyte-config/init/src/test/resources/connector_catalog.json +++ b/airbyte-config/init/src/test/resources/connector_catalog.json @@ -791,7 +791,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -799,16 +799,6 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 2e1aea25d215..e359a2331889 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -41,7 +41,7 @@ public class JdbcUtils { public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; public static final String PASSWORD_KEY = "password"; public static final String PORT_KEY = "port"; - public static final String TCP_PORT_KEY = "tcp-port"; + public static final List PORT_LIST_KEY = List.of("port"); public static final String SCHEMA_KEY = "schema"; // NOTE: this is the plural version of SCHEMA_KEY diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index 43d50c67daa4..80726e45e779 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -27,7 +27,6 @@ NORMALIZATION_TEST_MYSQL_DB_PORT = "NORMALIZATION_TEST_MYSQL_DB_PORT" NORMALIZATION_TEST_POSTGRES_DB_PORT = "NORMALIZATION_TEST_POSTGRES_DB_PORT" NORMALIZATION_TEST_CLICKHOUSE_DB_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_PORT" -NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT" NORMALIZATION_TEST_TIDB_DB_PORT = "NORMALIZATION_TEST_TIDB_DB_PORT" @@ -224,28 +223,20 @@ def setup_mssql_db(self): def setup_clickhouse_db(self): """ - ClickHouse official JDBC driver use HTTP port 8123, while Python ClickHouse - driver uses native port 9000, so we need to open both ports for destination - connector and dbt container respectively. + ClickHouse official JDBC driver uses HTTP port 8123. Ref: https://altinity.com/blog/2019/3/15/clickhouse-networking-part-1 """ start_db = True port = 8123 - tcp_port = 9000 if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT): port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT)) start_db = False - if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT): - tcp_port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT)) - start_db = False if start_db: port = self.find_free_port() - tcp_port = self.find_free_port() config = { "host": "localhost", "port": port, - "tcp-port": tcp_port, "database": self.target_schema, "username": "default", "password": "", @@ -263,8 +254,6 @@ def setup_clickhouse_db(self): "--ulimit", "nofile=262144:262144", "-p", - f"{config['tcp-port']}:9000", # Python clickhouse driver use native port - "-p", f"{config['port']}:8123", # clickhouse JDBC driver use HTTP port "-d", # so far, only the latest version ClickHouse server image turned on diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 756cdba77e9a..0adb06fe0c71 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -327,8 +327,6 @@ def transform_clickhouse(config: Dict[str, Any]): } if "password" in config: dbt_config["password"] = config["password"] - if "tcp-port" in config: - dbt_config["port"] = config["tcp-port"] return dbt_config @staticmethod diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 0294dd1fecfb..741bded1ba21 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -113,7 +113,6 @@ protected String getDefaultSchema(final JsonNode config) { protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db)) - .put(JdbcUtils.TCP_PORT_KEY, NATIVE_SECURE_PORT) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, USER_NAME) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 2238b3870b70..0cc91dd63bf1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,40 +27,30 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "tunnel_method": { "type": "object", diff --git a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md index 13d96b595110..c728bde55a2e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md +++ b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md @@ -10,12 +10,8 @@ This destination connector uses ClickHouse official JDBC driver, which uses HTTP ## Quick Notes -- ClickHouse JDBC driver uses HTTP protocal (default 8123) but [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) use TCP protocal (default 9000). - - This connector doesn't support nested streams and schema change yet. -- The community [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) has some bugs haven't been fixed yet, for example [https://github.com/silentsokolov/dbt-clickhouse/issues/20](https://github.com/silentsokolov/dbt-clickhouse/issues/20), so the dbt test is based on a fork [https://github.com/burmecia/dbt-clickhouse](https://github.com/burmecia/dbt-clickhouse). - ## API Reference The ClickHouse reference documents: [https://clickhouse.com/docs/en/](https://clickhouse.com/docs/en/) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index adf44db3857b..94f5acffb296 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,47 +27,37 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", "default": false, - "order": 7 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index b2688949c7db..0dfd1db487fb 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -89,15 +89,9 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() { - final Optional tcpPort = db.getExposedPorts().stream() - .map(exPort -> db.getMappedPort((Integer) exPort)) - .filter(el -> !db.getFirstMappedPort().equals(el)) - .findFirst(); - return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.TCP_PORT_KEY, tcpPort.get()) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 9cb7aabac298..a76e5983f8e1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -94,7 +94,7 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() throws Exception { return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME) - .put("schema", DB_NAME).put(JdbcUtils.TCP_PORT_KEY, 9000)); + .put("schema", DB_NAME)); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java index e935eff44730..9577ff5db275 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java @@ -34,7 +34,6 @@ public class ClickhouseDestinationSpecTest { + "\"username\" : \"clickhouse\", " + "\"database\" : \"clickhouse_db\", " + "\"port\" : 8123, " - + "\"tcp-port\" : 9000, " + "\"host\" : \"localhost\", " + "\"jdbc_url_params\" : \"property1=pValue1&property2=pValue2\", " + "\"ssl\" : true " diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index bf7dd37df0b9..7d0ccc711ce7 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -58,8 +58,7 @@ You will need to choose an existing database or create a new database that will You should now have all the requirements needed to configure ClickHouse as a destination in the UI. You'll need the following information to configure the ClickHouse destination: * **Host** -* **Port** (JDBC HTTP port, not the native port) -* **Tcp-port** (Native port, also required for data normalization) +* **Port** * **Username** * **Password** * **Database** From cd23430e5152f7f1b1d0c86b0a84def878fc4bc0 Mon Sep 17 00:00:00 2001 From: restrry Date: Thu, 22 Sep 2022 10:10:34 +0200 Subject: [PATCH 3/9] fix a typo in clickhouse icon --- .../src/main/resources/icons/{cliskhouse.svg => clickhouse.svg} | 0 .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/test/resources/connector_catalog.json | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename airbyte-config/init/src/main/resources/icons/{cliskhouse.svg => clickhouse.svg} (100%) diff --git a/airbyte-config/init/src/main/resources/icons/cliskhouse.svg b/airbyte-config/init/src/main/resources/icons/clickhouse.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/cliskhouse.svg rename to airbyte-config/init/src/main/resources/icons/clickhouse.svg diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index cf8f80cc6dfd..0963a00c3d40 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -70,7 +70,7 @@ dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - icon: cliskhouse.svg + icon: clickhouse.svg - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 09bae2809a4a..4049e0a6cbd5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -170,7 +170,7 @@ dockerRepository: airbyte/source-clickhouse dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse - icon: cliskhouse.svg + icon: clickhouse.svg sourceType: database releaseStage: alpha - name: Close.com diff --git a/airbyte-config/init/src/test/resources/connector_catalog.json b/airbyte-config/init/src/test/resources/connector_catalog.json index d4fd22c69112..c65a2d07cc35 100644 --- a/airbyte-config/init/src/test/resources/connector_catalog.json +++ b/airbyte-config/init/src/test/resources/connector_catalog.json @@ -6110,7 +6110,7 @@ "dockerRepository": "airbyte/source-clickhouse-strict-encrypt", "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse", - "icon": "cliskhouse.svg", + "icon": "clickhouse.svg", "sourceType": "database", "spec": { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", From ec94b2fe00e54506debe685d824a39d64d58d8b1 Mon Sep 17 00:00:00 2001 From: restrry Date: Thu, 22 Sep 2022 15:16:15 +0200 Subject: [PATCH 4/9] declare driver: http for dbt explicitly --- .../normalization/transform_config/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 0adb06fe0c71..87c88a0b0407 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -318,7 +318,7 @@ def transform_clickhouse(config: Dict[str, Any]): # https://docs.getdbt.com/reference/warehouse-profiles/clickhouse-profile dbt_config = { "type": "clickhouse", - "driver": "native", + "driver": "http", "host": config["host"], "port": config["port"], "schema": config["database"], From cf69749ae84c492977a39ce443aed479113ac43c Mon Sep 17 00:00:00 2001 From: restrry Date: Tue, 27 Sep 2022 10:04:26 +0200 Subject: [PATCH 5/9] roll back changes to auto genereates files --- .../seed/destination_definitions.yaml | 1 - .../resources/seed/destination_specs.yaml | 22 ++++++++++++++----- .../resources/seed/source_definitions.yaml | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 46b0a39d56e7..39f14d29f419 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -70,7 +70,6 @@ dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - icon: clickhouse.svg - name: Cloudflare R2 destinationDefinitionId: 0fb07be9-7c3b-4336-850d-5efc006152ee dockerRepository: airbyte/destination-r2 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index a8a71ab89a66..17e58c76911b 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -818,7 +818,7 @@ order: 0 port: title: "Port" - description: "HTTP port of the database." + description: "JDBC port (not the native port) of the database." type: "integer" minimum: 0 maximum: 65536 @@ -826,35 +826,45 @@ examples: - "8123" order: 1 + tcp-port: + title: "Native Port" + description: "Native port (not the JDBC) of the database." + type: "integer" + minimum: 0 + maximum: 65536 + default: 9000 + examples: + - "9000" + order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 2 + order: 3 username: title: "User" description: "Username to use to access the database." type: "string" - order: 3 + order: 4 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 4 + order: 5 jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" type: "string" - order: 5 + order: 6 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 6 + order: 7 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b9d4144881ff..929b8072502f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -178,7 +178,7 @@ dockerRepository: airbyte/source-clickhouse dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse - icon: clickhouse.svg + icon: cliskhouse.svg sourceType: database releaseStage: alpha - name: Close.com From e05cc2d2dcc07a5bb55b82d1d2e9b162a4bcb2fe Mon Sep 17 00:00:00 2001 From: restrry Date: Tue, 27 Sep 2022 10:04:52 +0200 Subject: [PATCH 6/9] bump destination clickhouse version --- .../connectors/destination-clickhouse-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- docs/integrations/destinations/clickhouse.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 50a0831ce3fe..8f27e4f6b615 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index f9eeeb4d8d7f..0f5a4cf4c350 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 7d0ccc711ce7..3ffb6ffb2459 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -80,6 +80,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:---------------------------------------------| +| 0.2.0 | 2022-09-27 | [16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters | | 0.1.12 | 2022-09-08 | [16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field | | 0.1.10 | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9 | | 0.1.8 | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout | From 5dcb619d6adccee92e4485fd7cd77348cfeacbeb Mon Sep 17 00:00:00 2001 From: grishick Date: Fri, 30 Sep 2022 15:14:58 -0700 Subject: [PATCH 7/9] fix unit test --- .../base-normalization/unit_tests/test_transform_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 337afaca4b12..2f0cff84eff6 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -440,7 +440,7 @@ def test_transform_clickhouse(self): actual = TransformConfig().transform_clickhouse(input) expected = { "type": "clickhouse", - "driver": "native", + "driver": "http", "host": "airbyte.io", "port": 9440, "schema": "default", From f33261a8e846f8f8d45b9daf0a192bbabfb8340b Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sat, 1 Oct 2022 02:47:20 +0000 Subject: [PATCH 8/9] auto-bump connector version [ci skip] --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 24 ++++++------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 10cb3ec2c6ab..1b25393b2108 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.12 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Cloudflare R2 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 050f9bb10ca5..a27d3317222a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -797,7 +797,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.12" +- dockerImage: "airbyte/destination-clickhouse:0.2.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -818,7 +818,7 @@ order: 0 port: title: "Port" - description: "JDBC port (not the native port) of the database." + description: "HTTP port of the database." type: "integer" minimum: 0 maximum: 65536 @@ -826,45 +826,35 @@ examples: - "8123" order: 1 - tcp-port: - title: "Native Port" - description: "Native port (not the JDBC) of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 9000 - examples: - - "9000" - order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 3 + order: 2 username: title: "User" description: "Username to use to access the database." type: "string" - order: 4 + order: 3 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 5 + order: 4 jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" type: "string" - order: 6 + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 7 + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" From d50192bd0b73c64469ca073a375cdfd8e8e514e6 Mon Sep 17 00:00:00 2001 From: grishick Date: Fri, 30 Sep 2022 21:16:59 -0700 Subject: [PATCH 9/9] Fix normalization test for Clickhouse Strict Encrypt --- .../normalization/transform_config/transform.py | 1 + .../bases/base-normalization/unit_tests/test_transform_config.py | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 87c88a0b0407..a762b39f1a45 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -319,6 +319,7 @@ def transform_clickhouse(config: Dict[str, Any]): dbt_config = { "type": "clickhouse", "driver": "http", + "verify": False, "host": config["host"], "port": config["port"], "schema": config["database"], diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 2f0cff84eff6..2adbb2f441cf 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -441,6 +441,7 @@ def test_transform_clickhouse(self): expected = { "type": "clickhouse", "driver": "http", + "verify": False, "host": "airbyte.io", "port": 9440, "schema": "default",