From 9449801a6dade9382c3a33207fdb299308cbdf98 Mon Sep 17 00:00:00 2001 From: Dorota Wojcik Date: Fri, 26 Jan 2024 15:37:39 +0100 Subject: [PATCH] add clickhouse credentials integrations docs --- .../concepts/data-integration-overview.md | 114 +++++++ .../howto/copy-data-across-instances.md | 18 +- .../howto/data-service-integration.md | 312 ++++++++++++------ sidebars.ts | 1 + src/components/ConsoleIcons/index.tsx | 6 + 5 files changed, 342 insertions(+), 109 deletions(-) create mode 100644 docs/products/clickhouse/concepts/data-integration-overview.md diff --git a/docs/products/clickhouse/concepts/data-integration-overview.md b/docs/products/clickhouse/concepts/data-integration-overview.md new file mode 100644 index 00000000..3b2901b8 --- /dev/null +++ b/docs/products/clickhouse/concepts/data-integration-overview.md @@ -0,0 +1,114 @@ +--- +title: Aiven for ClickHouse® data service integration +sidebar_label: Data integration +keywords: [data service integration, data source integration, managed credentials integration, managed databases integration, named collections] +--- + +import {ConsoleIcon} from "@site/src/components/ConsoleIcons"; + +Aiven for ClickHouse® supports different types of integration allowing you to efficiently connect with other services or data sources and access the data to be processed. + +There are a few ways of classifying integration types supported in Aiven for ClickHouse: + +- [By purpose](/docs/products/clickhouse/concepts/data-integration-overview#observability-integrations-vs-data-source-integrations): + observability integration vs data source integration +- [By location](/docs/products/clickhouse/concepts/data-integration-overview#integrations-between-aiven-managed-services-vs-external-integrations): + integration between Aiven-managed services vs external integration (between an + Aiven-managed service and an external service) +- By scope: [managed databases integration](/docs/products/clickhouse/concepts/data-integration-overview#managed-databases-integration) and + [managed credentials integration](/docs/products/clickhouse/concepts/data-integration-overview#managed-credentials-integration) + +## Observability integrations vs data source integrations + +Aiven for ClickHouse supports observability integrations and data source integrations, +which have different purposes: + +- [Observability integration](/docs/products/clickhouse/howto/list-integrations) is + connecting to other services (either Aiven-managed or external ones) to expose and + process logs and metrics. +- Data service integration is connecting to other services (either Aiven-managed or external) + to use them as data sources. In Aiven for ClickHouse, data service integration is + possible with the following data source types: + + - Apache Kafka® + - PostgreSQL® + - MySQL® + - ClickHouse® + - Amazon S3® + +## Integrations between Aiven-managed services vs external integrations + +By enabling data service integrations, you create streaming data pipelines across +services. Depending on where the services are located, you can have either +integrations between Aiven-managed services or external integrations (between an Aiven service +and an external data source or application). + +For integrating with external data sources, Aiven for ClickHouse provides two types of +data service integrations: + +- [Managed databases](/docs/products/clickhouse/concepts/data-integration-overview#managed-databases-integration) +- [Managed credentials](/docs/products/clickhouse/concepts/data-integration-overview#managed-credentials-integration) + +## Managed credentials integration + +The managed credentials integration uses the +[ClickHouse named collections](https://clickhouse.com/docs/en/operations/named-collections) +logic. It allows storing key-value pairs that are to be used as credentials when +connecting to external data sources. To integrate data, you create tables using table +engines. With the managed credentials integration enabled, +querying the data is easier and quicker since you no longer need connections parameters in +each query. They are stored and available from credential storage in Aiven. + +Managed credentials integration in Aiven for ClickHouse is supported with the following +data source types: + +- PostgreSQL® +- MySQL® +- ClickHouse® +- Amazon S3® + +:::important +The managed credentials integration works one-way: It allows to integrate with data source +and access your data from Aiven for ClickHouse, but it doesn’t allow to access Aiven for +ClickHouse data from the integrated data source. +::: + +For information on how table engines work in Aiven for ClickHouse services, preview +[Engines: database and table](/docs/products/clickhouse/concepts/service-architecture#engines-database-and-table). +For the list of table engines available in Aiven for ClickHouse, check +[Supported table engines](/docs/products/clickhouse/reference/supported-table-engines). + +## Managed databases integration + +The managed databases integration allows using a database engine for handling your +external data. When enabled, this type of integration provides you with an automatically +created database, where the remote data is exposed. + +Managed databases integration in Aiven for ClickHouse is supported with the following +data source types: + +- PostgreSQL® +- Apache Kafka® + +For information on how database engines work in Aiven for ClickHouse services, preview +[Engines: database and table](/docs/products/clickhouse/concepts/service-architecture#engines-database-and-table). +For more information on ClickHouse database engines, see +[Database engines](https://clickhouse.com/docs/en/engines/database-engines). + +## Supported data source types + +Depending on a data source type, Aiven for ClickHouse supports different integration modes. + +| Data source type | Data source integration
(with Aiven service
or external source)| Managed databases integration| Managed credentials integration | +|------------------|----------------------------|------------------------------|---------------------------------| +| PostgreSQL | | | | +| MySQL | | | | +| Apache Kafka | | | | +| ClickHouse | | | | +| Amazon S3 | | | | + +## Related pages + +- [Set up Aiven for ClickHouse® data service integration](/docs/products/clickhouse/howto/data-service-integration) +- [Manage Aiven for ClickHouse® integration databases](/docs/products/clickhouse/howto/integration-databases) +- [Integrate your Aiven for ClickHouse® service](/docs/products/clickhouse/howto/list-integrations) diff --git a/docs/products/clickhouse/howto/copy-data-across-instances.md b/docs/products/clickhouse/howto/copy-data-across-instances.md index f81b7808..89206485 100644 --- a/docs/products/clickhouse/howto/copy-data-across-instances.md +++ b/docs/products/clickhouse/howto/copy-data-across-instances.md @@ -15,16 +15,28 @@ Details of the source (remote) server ## Copy data -1. From your target servers, use the `remoteSecure()` +1. From your target server, use the `remoteSecure()` function to select data from the source server. - ```shell + ```sql SELECT * FROM remoteSecure('HOSTNAME:PORT', db.remote_engine_table, 'USERNAME', 'PASSWORD') LIMIT 3; ``` + :::tip + If you have the + [managed credentials integration](/docs/products/clickhouse/concepts/data-integration-overview#managed-credentials-integration) + enabled, you can use instead: + + ```sql + SELECT * FROM remoteSecure('service_YOUR_REMOTE_CLUSTER', db.remote_engine_table) LIMIT 3; + ``` + + See how to [enable the managed credentials integration](/docs/products/clickhouse/howto/data-service-integration#integrate-with-external-data-sources). + ::: + 1. Insert the selected data into the target server. - ```shell + ```sql INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... ``` diff --git a/docs/products/clickhouse/howto/data-service-integration.md b/docs/products/clickhouse/howto/data-service-integration.md index f8d20921..0e1f15e5 100644 --- a/docs/products/clickhouse/howto/data-service-integration.md +++ b/docs/products/clickhouse/howto/data-service-integration.md @@ -1,136 +1,236 @@ --- -title: Manage Aiven for ClickHouse® data service integrations +title: Set up Aiven for ClickHouse® data service integrations +sidebar_label: Integrate with data source +keywords: [data service integration, data source integration, managed credentials integration, managed databases integration, named collections] --- -Aiven for ClickHouse supports two types of integrations: +import {ConsoleIcon} from "@site/src/components/ConsoleIcons"; +import ConsoleLabel from "@site/src/components/ConsoleIcons"; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; -- Regular integrations: Logs, metrics, dataflow/replication, and - authentication integrations among your Aiven services and with - external applications -- Data service integrations: Integrations with other Aiven services to - use them as data sources +Connect your Aiven for ClickHouse® service with another Aiven-managed service or external data source to make your data available in the Aiven for ClickHouse service. -For information on how to set up and use regular integrations in Aiven -for ClickHouse, see [Integrate your Aiven for ClickHouse® service](/docs/products/clickhouse/howto/list-integrations). - -## About data service integrations - -By enabling data service integrations in Aiven for ClickHouse, you -create streaming data pipelines across services. Aiven for ClickHouse -supports data service integrations with Aiven for Kafka® and Aiven for -PostgreSQL®. - -You can create Aiven for ClickHouse® data service integrations in the -[Aiven Console](https://console.aiven.io/). - -:::note[Integration databases] -When creating integrations in the **Data service integrations** wizard, -you can also create integration databases connected to the services you -are integrating with. - -If you prefer to create a data service integration without adding -integration databases, you can create integration databases for your -service any time later. See -[Manage Aiven for ClickHouse® integration databases](/docs/products/clickhouse/howto/integration-databases). +:::tip +Read about data service integration types and methods available with Aiven for +ClickHouse® in +[Aiven for ClickHouse® data service integration](/docs/products/clickhouse/concepts/data-integration-overview). ::: ## Prerequisites -- Aiven account -- Access to [Aiven Console](https://console.aiven.io/) +- You are familiar with the limitations listed in + [About Aiven for ClickHouse® data service integration](/docs/products/clickhouse/concepts/data-integration-overview#supported-data-source-types). +- You have an organization, a project, and an Aiven for ClickHouse service in Aiven. +- You have access to the [Aiven Console](https://console.aiven.io/). ## Create data service integrations {#create-data-service-integration} -1. Log in to the [Aiven Console](https://console.aiven.io/). - -1. In the **Services** page, select an Aiven for ClickHouse service you - want to integrate with a data service. - -1. Select **Get started** from the **Integrate your Aiven for - ClickHouse** section in the **Overview** page of your service. - -1. In the **Data service integrations** wizard, select one of the - following options: - - +1. Log in to the [Aiven Console](https://console.aiven.io/), and go to an organization + and a project. +1. From , select an Aiven for ClickHouse service to integrate + with a data source. +1. In the service , go to **Data pipeline**. -
- Option 1: Create a service and integrate it + - To create the first integration for that service, click **Add data source**. + - To add an integrtion for that service, click . - To create an integration with a **new service**: + As a result, the **Data service integrations** wizard opens, showing + a list of all data sources available for integration. If there are no data sources to + integrate with, create an Aiven-managed service or an integration endpoint by + clicking **Create service** or **Add external endpoint** respectively. - 1. Make sure the checkboxes for both service types are unchecked. - 1. In the **Data service integrations** view, select **Create - service**. - 1. [Set up the new service](/docs/platform/howto/create_new_service). - 1. Come back to your primary service and create an integration to - the newly created service. - 1. Skip the steps that follow and start over with building your integration using this - instruction but follow **Option 2: Use an existing service and integrate it**. -
- -
- Option 2: Use an existing service and integrate it +1. Depending on your use case, continue as instructed in either + [Integrate with Aiven-managed services](#integrate-aiven-services) or + [Integrate with external data sources](#integrate-with-external-data-sources). - To create an integration with an **existing service**: - - 1. Select a service of the chosen type from the list of services - available for integration. - 1. Select **Continue** and proceed to the next step to integrate - the database. - -
- -1. In the **Integration databases** view, select either **Enable - without databases** or **Add databases** depending on whether you - want to enable your integration with databases: - - - To enable your integration **with** databases: - 1. In the **Integration databases** view, select **Add - databases**. - - 1. In the **Add integration databases** section, enter database - names and schema names and select **Enable** when ready. - - You can preview the created databases by selecting - **Databases tables** from the sidebar. - - To enable your integration **without** databases - 1. In the **Integration databases** view, select **Enable - without databases**. - - You can preview the created integration by selecting - **Overview** from the sidebar. +### Integrate with Aiven-managed services {#integrate-aiven-services} + +You can integrate either with a new Aiven-managed service or with an existing Aiven-managed +services: + + + +To create an integration with a new service in the **Data service integrations** wizard: + +1. Unselect all available data **Service types**. +1. Click **Create service**. +1. [Set up the new service](/docs/platform/howto/create_new_service). +1. Come back to your Aiven for ClickHouse service ( > + **Data pipeline** > **Add data source**), and create an integration to the newly created + service following the **Integrate with an existing service** tab. + + +To create an integration with an existing service in the **Data service integrations** wizard: + +1. Select **Service types** of Aiven-managed services to integrate with. See + [supported data source types](/docs/products/clickhouse/concepts/data-integration-overview#supported-data-source-types) + for supported service types. +1. Select **Data service** of the chosen type from the list of services available for + integration, and click **Continue**. +1. Click either **Enable without databases** or **Add databases**. + + :::note + If you prefer to create a data service integration without adding integration databases, + you can create integration databases for your service any time later. See + [Manage Aiven for ClickHouse® integration databases](/docs/products/clickhouse/howto/integration-databases). + :::: + + - Enable your integration with databases: + + Click **Add databases**, enter database and schema names, and click **Enable**. + + You can preview the created databases by selecting + from the sidebar. + + - Enable your integration without databases: + + Click **Enable without databases**. + + To preview the created integration, from the sidebar, click . + + + +### Integrate with external data sources + +For integration with external data sources, Aiven for ClickHouse offers two methods: + +- [Managed databases](/docs/products/clickhouse/concepts/data-integration-overview.md#managed-databases-integration) +- [Managed credentials](/docs/products/clickhouse/concepts/data-integration-overview.md#managed-credentials-integration) + +To create an integration with an external data source in the **Data service integrations** +wizard: + +1. Select **Service types** of an external data source to integrate with. See + [supported data source types](/docs/products/clickhouse/concepts/data-integration-overview#supported-data-source-types) + for supported service types. +1. Select an external **Data service** of the chosen type from the list of services available + for integration, and click **Continue**. +1. Select an integration method, either **Managed databases** or **Managed credentials**, + and click **Continue**. +1. Integrate with managed databases or with managed credentials: + + + +The managed databases integration uses databases engines and, when enabled, +automatically creates databases in your Aiven for ClickHouse, where you can ingest your +external data. + +1. Select **Managed databases**, and click **Continue**. +1. Click **Add databases** to have custom databases created along with the integration, or + click **Enable without databases** to integrate with no custom databases created. + + :::tip + If you choose to have databases created automatically, you can query them using a + statement similar to the following: + + ```sql + SELECT * + FROM EXTERNAL_POSTGRESQL_RESOURCE_NAME.POSTGRESQL_TABLE_NAME + ``` + + ::: + + + +The managed credentials integration supports storing connection parameters in Aiven +and allows you to create tables for your external data. The stored connection parameters +are automatically seeded in your external data queries. + +1. Select **Managed credentials**, and click **Continue** > **Enable**. +1. Create tables using + [table engines](/docs/products/clickhouse/reference/supported-table-engines), for + example the PostgreSQL engine: + + ```sql + CREATE TABLE default.POSTGRESQL_TABLE_NAME + ( + `float_nullable` Nullable(Float32), + `str` String, + `int_id` Int32 + ) + ENGINE = PostgreSQL(postgres_creds); + ``` + + :::tip + For details on how to use different table engines for integrations with external + systems, see the + [upstream ClickHouse documentation](https://clickhouse.com/docs/en/engines/table-engines/integrations). + ::: + +Depending on the type of data source you are integrated with, you can access your credentials +storage by passing your data source name in the following query: + +```sql +# PostgreSQL data source + +SELECT * +FROM postgresql( + `service_POSTGRESQL_SOURCE_NAME`, + database='defaultdb', + table='tables', + schema='information_schema' +) +``` + +```sql +# MySQL data source + +SELECT * +FROM mysql( + `service_MYSQL_SOURCE_NAME`, + database='mysql', + table='slow_log' +) +``` + +```sql +# Amazon S3 data source + +SELECT * FROM s3( + `endpoint_S3_SOURCE_NAME`, + filename='*.csv', + format='CSVWithNames') +``` + +:::warning +When you try to run a managed credentials query with a typo, the query fails with an +error message related to grants. +::: + + ## View data service integrations -1. Log in to the [Aiven Console](https://console.aiven.io/). -1. In the **Services** page, select an Aiven for ClickHouse service you - want to check integrations for. -1. In the **Overview** page of your service, find the **Data service - integration** section at the top and discover your integrations - grouped according to service types (PostgreSQL or Apache Kafka). -1. Select the meatball menu for a particular service group to preview - active data service integrations within that group. +1. Log in to the [Aiven Console](https://console.aiven.io/), and go to an organization + and a project. +1. From , select an Aiven for ClickHouse service to display + integrations for. +1. In the service , go to **Data pipeline** + and find your integrations grouped according to service types. +1. Click next to a particular service group to preview + active data service integrations within that group. ## Stop data service integrations -1. Log in to the [Aiven Console](https://console.aiven.io/). -1. In the **Services** page, select an Aiven for ClickHouse service you +1. Log in to the [Aiven Console](https://console.aiven.io/), and go to an organization + and a project. +1. From , select an Aiven for ClickHouse service you want to stop integrations for. -1. In the **Overview** page of your service, find the **Data service - integration** section at the top and select the meatball menu for a - service group that your unwanted integration belongs to. +1. In the service , go to **Data pipeline** and click + next to a service group that the integration to be + stopped belongs to. 1. From the **Active data service integrations** list, select the - service integration that you no longer need and select **Disconnect - integration**. -1. In the **Warning** popup, study the impact of disconnecting from a - service and select **Disconnect integration** if you accept erasing - all the databases and configuration information. + service integration that you no longer need, and click **Disconnect integration**. +1. Evaluate the impact of disconnecting from a service explained in + **Warning**, and click **Disconnect integration** if you accept erasing all the + databases and configuration information. -Your integration has been removed along with all the corresponding -databases and configuration information. +As a result, your integration is terminated and all the corresponding databases and +configuration information are deleted. ## Related pages +- [Aiven for ClickHouse® data service integration](/docs/products/clickhouse/concepts/data-integration-overview) - [Manage Aiven for ClickHouse® integration databases](/docs/products/clickhouse/howto/integration-databases) - [Integrate your Aiven for ClickHouse® service](/docs/products/clickhouse/howto/list-integrations) diff --git a/sidebars.ts b/sidebars.ts index e944a0ac..3eb2b7d2 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1320,6 +1320,7 @@ const sidebars: SidebarsConfig = { 'products/clickhouse/concepts/strings', 'products/clickhouse/concepts/federated-queries', 'products/clickhouse/concepts/clickhouse-tiered-storage', + 'products/clickhouse/concepts/data-integration-overview', ], }, { diff --git a/src/components/ConsoleIcons/index.tsx b/src/components/ConsoleIcons/index.tsx index 78afdb8e..bfc0fcff 100644 --- a/src/components/ConsoleIcons/index.tsx +++ b/src/components/ConsoleIcons/index.tsx @@ -356,6 +356,12 @@ export default function ConsoleLabel({name}): ReactElement { Add configuration options ); + case 'addnew': + return ( + <> + Add new + + ); case 'plus': return ( <>