cdc: Add integration tests with Debezium (#10310)

ref #1799
pingcap · Feb 19, 2024 · 7f59c70 · 7f59c70
1 parent 0850a1b
commit 7f59c70
Show file tree

Hide file tree

Showing 42 changed files with 2,687 additions and 28 deletions.
diff --git a/tests/integration_tests/_utils/start_tidb_cluster_impl b/tests/integration_tests/_utils/start_tidb_cluster_impl
@@ -190,7 +190,6 @@ if [[ "$tidb_config" != "" ]]; then
 else
 	cat - >"$OUT_DIR/tidb-config.toml" <<EOF
 split-table = true
-alter-primary-key = true
 new_collations_enabled_on_first_bootstrap = true
 EOF
 fi

diff --git a/tests/integration_tests/debezium/README.md b/tests/integration_tests/debezium/README.md
@@ -0,0 +1,53 @@
+# Debezium Integration Test Locally
+
+This file shows how to run debezium integration test locally
+
+```
+cd tiflow/tests/integration_tests/debezium
+docker compose up
+```
+
+```
+curl -i -X POST \
+  -H "Accept:application/json" \
+  -H "Content-Type:application/json" \
+  localhost:8083/connectors/ --data-binary @- << EOF
+{
+  "name": "my-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
+    "tasks.max": "1",
+    "database.hostname": "mysql",
+    "database.port": "3306",
+    "database.user": "debezium",
+    "database.password": "dbz",
+    "database.server.id": "184054",
+    "topic.prefix": "dbserver1",
+    "schema.history.internal.kafka.bootstrap.servers": "kafka:9092",
+    "schema.history.internal.kafka.topic": "schemahistory.test",
+    "transforms": "x",
+    "transforms.x.type": "org.apache.kafka.connect.transforms.RegexRouter",
+    "transforms.x.regex": "(.*)",
+    "transforms.x.replacement":"output_debezium",
+    "binary.handling.mode": "base64",
+    "decimal.handling.mode": "double"
+  }
+}
+EOF
+```
+
+```
+tiup playground nightly --tiflash 0 --ticdc 1 
+```
+
+```
+tiup cdc cli changefeed create \
+  --server=http://127.0.0.1:8300 \
+  --sink-uri="kafka://127.0.0.1:9094/output_ticdc?protocol=debezium&kafka-version=2.4.0"
+```
+
+```
+go run ./src --db.mysql="root@tcp(127.0.0.1:3306)/{db}?allowNativePasswords=true" --cdc.kafka 127.0.0.1:9094
+```
+
+
diff --git a/tests/integration_tests/debezium/docker-compose.yml b/tests/integration_tests/debezium/docker-compose.yml
@@ -0,0 +1,65 @@
+version: "2"
+services:
+  zookeeper:
+    restart: always
+    image: quay.io/debezium/zookeeper:2.4
+    ports:
+      - 2181:2181
+      - 2888:2888
+      - 3888:3888
+  kafka:
+    restart: always
+    image: quay.io/debezium/kafka:2.4
+    ports:
+      - 9092:9092
+      - 9094:9094
+    depends_on:
+      - zookeeper
+    environment:
+      - ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LISTENERS=INTERNAL://0.0.0.0:9092,OUTSIDE://0.0.0.0:9094
+      - KAFKA_ADVERTISED_LISTENERS=INTERNAL://kafka:9092,OUTSIDE://localhost:9094
+      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
+      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
+  mysql:
+    restart: always
+    ports:
+      - 3306:3306
+    image: quay.io/debezium/example-mysql:2.4
+    environment:
+      - MYSQL_ROOT_PASSWORD=
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_USER=mysqluser
+      - MYSQL_PASSWORD=mysqlpw
+  connect:
+    restart: always
+    image: quay.io/debezium/connect:2.4
+    ports:
+      - 8083:8083
+    depends_on:
+      - kafka
+      - mysql
+    environment:
+      - BOOTSTRAP_SERVERS=kafka:9092
+      - GROUP_ID=1
+      - CONFIG_STORAGE_TOPIC=my_connect_configs
+      - OFFSET_STORAGE_TOPIC=my_connect_offsets
+      - STATUS_STORAGE_TOPIC=my_connect_statuses
+  # watcher_dbz: # For Debug Purpose
+  #   restart: always
+  #   image: quay.io/debezium/kafka:2.4
+  #   depends_on:
+  #     - kafka
+  #   command: watch-topic -a -k output_debezium
+  #   environment:
+  #     - ZOOKEEPER_CONNECT=zookeeper:2181
+  #     - KAFKA_BROKER=kafka:9092
+  # watcher_ticdc: # For Debug Purpose
+  #   restart: always
+  #   image: quay.io/debezium/kafka:2.4
+  #   depends_on:
+  #     - kafka
+  #   command: watch-topic -a -k output_ticdc
+  #   environment:
+  #     - ZOOKEEPER_CONNECT=zookeeper:2181
+  #     - KAFKA_BROKER=kafka:9092
diff --git a/tests/integration_tests/debezium/go.mod b/tests/integration_tests/debezium/go.mod
@@ -0,0 +1,84 @@
+module github.com/breezewish/checker
+
+go 1.21.0
+
+require (
+	github.com/go-sql-driver/mysql v1.7.1
+	github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873
+	github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4
+	github.com/thessem/zap-prettyconsole v0.3.0
+	go.uber.org/zap v1.26.0
+)
+
+require (
+	github.com/Code-Hex/dd v1.1.0 // indirect
+	github.com/alecthomas/chroma v0.10.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudfoundry/gosigar v1.3.6 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.1.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/klauspost/compress v1.17.1 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect
+	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
+	github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 // indirect
+	github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 // indirect
+	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/prometheus/client_golang v1.17.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/segmentio/kafka-go v0.4.45 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.10 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
+	github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e // indirect
+	github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/twmb/murmur3 v1.1.6 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.10 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/grpc v1.59.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+)