Release 2.5.0

.github/workflows/testflows-sink-connector-kafka.yml

@@ -11,7 +11,7 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
+        default: docker://clickhouse/clickhouse-server:23.8
       output_format:
         description: "Testflows output style."
         type: string
@@ -34,7 +34,7 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
+        default: docker://clickhouse/clickhouse-server:23.8
       extra_args:
         description: "Specific Suite To Run (Default * to run everything)."
         required: false

pom.xml

@@ -14,7 +14,7 @@
         <maven.compiler.target>17</maven.compiler.target>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <version.debezium>2.5.0.Beta1</version.debezium>
+        <version.debezium>2.7.2.Final</version.debezium>
         <version.junit>5.9.1</version.junit>
         <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

sink-connector-lightweight/config.examples/mysql_config.yaml

@@ -12,11 +12,11 @@ database.allowPublicKeyRetrieval: "true"
 snapshot.mode: "schema_only"
 connector.class: "io.debezium.connector.mysql.MySqlConnector"
 offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-offset.storage.offset.storage.jdbc.offset.table.name: "replica_source_info"
+offset.storage.jdbc.offset.table.name: "replica_source_info"
 offset.storage.jdbc.url: "jdbc:clickhouse://localhost:8123"
 offset.storage.jdbc.user: "root"
 offset.storage.jdbc.password: "root"
-offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE %s
+offset.storage.jdbc.offset.table.ddl: "CREATE TABLE %s
 (
     `id` String,
     `offset_key` String,
@@ -28,7 +28,7 @@ offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE %s
 ENGINE = ReplacingMergeTree(_version)
 ORDER BY id
 SETTINGS index_granularity = 8192"
-offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
+offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
 schema.history.internal: io.debezium.storage.jdbc.history.JdbcSchemaHistory
 schema.history.internal.jdbc.url: "jdbc:clickhouse://localhost:8123"
 schema.history.internal.jdbc.user: "root"

sink-connector-lightweight/dependency-reduced-pom.xml

@@ -131,7 +131,7 @@
     <dependency>
       <groupId>io.debezium</groupId>
       <artifactId>debezium-connector-mongodb</artifactId>
-      <version>2.7.0.Beta2</version>
+      <version>3.0.0.Final</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -140,6 +140,46 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>io.debezium</groupId>
+      <artifactId>debezium-testing-testcontainers</artifactId>
+      <version>3.0.0.Final</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>kafka</artifactId>
+          <groupId>org.testcontainers</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>awaitility</artifactId>
+          <groupId>org.awaitility</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>mssqlserver</artifactId>
+          <groupId>org.testcontainers</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>oracle-xe</artifactId>
+          <groupId>org.testcontainers</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>junit-platform-launcher</artifactId>
+          <groupId>org.junit.platform</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>junit-jupiter</artifactId>
+          <groupId>org.junit.jupiter</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>quarkus-test-common</artifactId>
+          <groupId>io.quarkus</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>okhttp</artifactId>
+          <groupId>com.squareup.okhttp3</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.mongodb</groupId>
       <artifactId>mongo-java-driver</artifactId>
@@ -273,14 +313,14 @@
           <artifactId>junit-jupiter-params</artifactId>
           <groupId>org.junit.jupiter</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>junit-platform-launcher</artifactId>
-          <groupId>org.junit.platform</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>junit-platform-launcher</artifactId>
+          <groupId>org.junit.platform</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -318,6 +358,7 @@
     <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
     <version.testcontainers>1.19.1</version.testcontainers>
     <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
+    <version.kafka>3.8.0</version.kafka>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <sink-connector-library-version>0.0.9</sink-connector-library-version>
     <javalin-version>5.5.0</javalin-version>
@@ -326,7 +367,7 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
     <maven.compiler.target>17</maven.compiler.target>
-    <version.debezium>2.7.0.Beta2</version.debezium>
+    <version.debezium>3.0.0.Final</version.debezium>
     <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
   </properties>
 </project>

sink-connector-lightweight/docker/config.yml

@@ -130,7 +130,7 @@ database.connectionTimeZone: "UTC"
 
 # Configuration to override the clickhouse database name for a given MySQL database name. If this configuration is not
 # provided, the MySQL database name will be used as the ClickHouse database name.
-#clickhouse.database.override.map: "employees:employees2, products:productsnew
+clickhouse.database.override.map: "test:ch_test"
 
 # clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
 #clickhouse.datetime.timezone: "UTC"
@@ -147,6 +147,9 @@ database.connectionTimeZone: "UTC"
 #disable.ddl: If set to true, the connector will ignore DDL events. The default is false.
 #disable.ddl: "false"
 
+#ignore.ddl.regex: If set, the connector will ignore DDL events that match the regex.
+ignore.ddl.regex: "(?i)(ANALYZE PARTITION).*"
+
 #disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false.
 #disable.drop.truncate: "false"
 

sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml

@@ -23,11 +23,11 @@ data:
     offset.flush.interval.ms: 5000
     connector.class: "io.debezium.connector.postgresql.PostgresConnector"
     offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-    offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
+    offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
     offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
     offset.storage.jdbc.user: "root"
     offset.storage.jdbc.password: "root"
-    offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
+    offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
     (
       `id` String,
       `offset_key` String,
@@ -39,15 +39,7 @@ data:
     ENGINE = ReplacingMergeTree(_version)
     ORDER BY id
     SETTINGS index_granularity = 8192"
-    offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
-    schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-    schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-    schema.history.internal.jdbc.user: "root"
-    schema.history.internal.jdbc.password: "root"
-    schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
-    (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-
-    schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
+    offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
     enable.snapshot.ddl: "true"
     auto.create.tables: "true"
     database.dbname: "public"
@@ -75,11 +67,11 @@ data:
       offset.flush.interval.ms: "5000"
       connector.class: "io.debezium.connector.mysql.MySqlConnector"
       offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-      offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
+      offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
       offset.storage.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
       offset.storage.jdbc.user: "root"
       offset.storage.jdbc.password: "secretsecret"
-      offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
+      offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
         (
           `id` String,
           `offset_key` String,
@@ -91,7 +83,7 @@ data:
       ENGINE = ReplacingMergeTree(_version)
       ORDER BY id
       SETTINGS index_granularity = 8192"
-      offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
+      offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
       schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
       schema.history.internal.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
       schema.history.internal.jdbc.user: "root"

sink-connector-lightweight/pom.xml

@@ -13,7 +13,7 @@
         <maven.compiler.target>17</maven.compiler.target>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <version.debezium>2.7.2.Final</version.debezium>
+        <version.debezium>3.0.0.Final</version.debezium>
         <version.junit>5.9.1</version.junit>
         <version.testcontainers>1.19.1</version.testcontainers>
         <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
@@ -24,6 +24,7 @@
         <quarkus.platform.version>2.14.0.Final</quarkus.platform.version>
         <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
         <sink-connector-library-version>0.0.9</sink-connector-library-version>
+        <version.kafka>3.8.0</version.kafka>
         <javalin-version>5.5.0</javalin-version>
     </properties>
     <dependencyManagement>
@@ -142,6 +143,13 @@
             <version>${version.debezium}</version>
             <scope>test</scope>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/io.debezium/debezium-testing-testcontainers -->
+        <dependency>
+            <groupId>io.debezium</groupId>
+            <artifactId>debezium-testing-testcontainers</artifactId>
+            <version>${version.debezium}</version>
+            <scope>test</scope>
+        </dependency>
 
 
         <dependency>
@@ -170,7 +178,7 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
-            <version>3.3.1</version>
+            <version>${version.kafka}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -336,7 +344,7 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>connect-api</artifactId>
-            <version>2.8.1</version>
+            <version>${version.kafka}</version>
             <!-- <scope>provided</scope> -->
             <exclusions>
                 <exclusion>
@@ -414,7 +422,7 @@
             <!-- or clickhouse-grpc-client if you prefer gRPC -->
             <artifactId>clickhouse-jdbc</artifactId>
             <classifier>shaded</classifier>
-            <version>0.6.0</version>
+            <version>0.6.5</version>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -494,13 +502,13 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>connect-json</artifactId>
-            <version>3.1.0</version>
+            <version>${version.kafka}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>connect-json</artifactId>
-            <version>3.1.0</version>
+            <version>${version.kafka}</version>
             <scope>compile</scope>
         </dependency>