diff --git a/checker/cmd.go b/checker/cmd.go index 23d0f3be56..cf830f1870 100644 --- a/checker/cmd.go +++ b/checker/cmd.go @@ -39,8 +39,19 @@ func CheckSyncConfig(ctx context.Context, cfgs []*config.SubTaskConfig) error { return nil } - // all `IgnoreCheckingItems` of sub-task are same, so we take first one - checkingItems := config.FilterCheckingItems(cfgs[0].IgnoreCheckingItems) + // all `IgnoreCheckingItems` and `Mode` of sub-task are same, so we take first one + // for ModeFull we don't need replication privilege; for ModeIncrement we don't need dump privilege + ignoreCheckingItems := cfgs[0].IgnoreCheckingItems + // we directly append ignore checking items here which may cause duplicate in ignoreCheckingItems + // but in config.FilterCheckingItems we only use this to delete map's keys so it is tolerable to append directly here + switch cfgs[0].Mode { + case config.ModeFull: + ignoreCheckingItems = append(ignoreCheckingItems, config.ReplicationPrivilegeChecking, + config.BinlogEnableChecking, config.BinlogFormatChecking, config.BinlogRowImageChecking) + case config.ModeIncrement: + ignoreCheckingItems = append(ignoreCheckingItems, config.DumpPrivilegeChecking) + } + checkingItems := config.FilterCheckingItems(ignoreCheckingItems) if len(checkingItems) == 0 { return nil } diff --git a/cmd/dm-master/main.go b/cmd/dm-master/main.go index 66d6e3ff6d..ef3bc41745 100644 --- a/cmd/dm-master/main.go +++ b/cmd/dm-master/main.go @@ -39,7 +39,7 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - fmt.Printf("parse cmd flags err %s", err) + fmt.Printf("parse cmd flags err %s", errors.ErrorStack(err)) os.Exit(2) } diff --git a/cmd/dm-tracer/main.go b/cmd/dm-tracer/main.go index fefdd22315..8ca984f14d 100644 --- a/cmd/dm-tracer/main.go +++ b/cmd/dm-tracer/main.go @@ -36,7 +36,7 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - fmt.Printf("parse cmd flags err %s", err) + fmt.Printf("parse cmd flags err %s", errors.ErrorStack(err)) os.Exit(2) } diff --git a/cmd/dm-worker/main.go b/cmd/dm-worker/main.go index 64500bf8c8..657b4bb969 100644 --- a/cmd/dm-worker/main.go +++ b/cmd/dm-worker/main.go @@ -41,7 +41,7 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - fmt.Printf("parse cmd flags err: %s", err) + fmt.Printf("parse cmd flags err: %s", errors.ErrorStack(err)) os.Exit(2) } diff --git a/tests/_utils/check_count b/tests/_utils/check_count new file mode 100755 index 0000000000..08a36402a3 --- /dev/null +++ b/tests/_utils/check_count @@ -0,0 +1,12 @@ +#!/bin/sh + +set -eu + +count=$(grep -c "$1" "$TEST_DIR/sql_res.$TEST_NAME.txt") +if [ $count != $2 ]; then + echo "TEST FAILED: OUTPUT CONTAINS '$1' for $count times, which supposed to be $2 times" + echo "____________________________________" + cat "$TEST_DIR/sql_res.$TEST_NAME.txt" + echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" + exit 1 +fi diff --git a/tests/full_mode/conf/diff_config.toml b/tests/full_mode/conf/diff_config.toml new file mode 100644 index 0000000000..7aa11c6367 --- /dev/null +++ b/tests/full_mode/conf/diff_config.toml @@ -0,0 +1,58 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 1000 + +check-thread-count = 4 + +sample-percent = 100 + +use-rowid = false + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "full_mode" +tables = ["~t.*"] + +[[table-config]] +schema = "full_mode" +table = "t1" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "full_mode" +table = "t1" + +[[table-config]] +schema = "full_mode" +table = "t2" + +[[table-config.source-tables]] +instance-id = "source-2" +schema = "full_mode" +table = "t2" + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" +instance-id = "source-1" + +[[source-db]] +host = "127.0.0.1" +port = 3307 +user = "root" +password = "" +instance-id = "source-2" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" diff --git a/tests/full_mode/conf/dm-master.toml b/tests/full_mode/conf/dm-master.toml new file mode 100644 index 0000000000..334e0de993 --- /dev/null +++ b/tests/full_mode/conf/dm-master.toml @@ -0,0 +1,9 @@ +# Master Configuration. + +[[deploy]] +source-id = "mysql-replica-01" +dm-worker = "127.0.0.1:8262" + +[[deploy]] +source-id = "mysql-replica-02" +dm-worker = "127.0.0.1:8263" diff --git a/tests/full_mode/conf/dm-task.yaml b/tests/full_mode/conf/dm-task.yaml new file mode 100644 index 0000000000..63c621a195 --- /dev/null +++ b/tests/full_mode/conf/dm-task.yaml @@ -0,0 +1,51 @@ +--- +name: test +task-mode: full +is-sharding: false +meta-schema: "dm_meta" +remove-meta: false +enable-heartbeat: true +heartbeat-update-interval: 1 +heartbeat-report-interval: 1 +timezone: "Asia/Shanghai" + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + password: "" + +mysql-instances: + - source-id: "mysql-replica-01" + black-white-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + + - source-id: "mysql-replica-02" + black-white-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + +black-white-list: + instance: + do-dbs: ["full_mode"] + +mydumpers: + global: + mydumper-path: "./bin/mydumper" + threads: 4 + chunk-filesize: 64 + skip-tz-utc: true + extra-args: "" + +loaders: + global: + pool-size: 16 + dir: "./dumped_data" + +syncers: + global: + worker-count: 16 + batch: 100 diff --git a/tests/full_mode/conf/dm-worker1.toml b/tests/full_mode/conf/dm-worker1.toml new file mode 100644 index 0000000000..561840a022 --- /dev/null +++ b/tests/full_mode/conf/dm-worker1.toml @@ -0,0 +1,15 @@ +# Worker Configuration. + +server-id = 101 # we don't give dm_full account REPLICATION SLAVE privilege here, so we must specify server-id here +source-id = "mysql-replica-01" +flavor = "" +enable-gtid = false +relay-binlog-name = "" +relay-binlog-gtid = "" + +[from] +host = "127.0.0.1" +user = "dm_full" +password = "" +port = 3306 + diff --git a/tests/full_mode/conf/dm-worker2.toml b/tests/full_mode/conf/dm-worker2.toml new file mode 100644 index 0000000000..b338074100 --- /dev/null +++ b/tests/full_mode/conf/dm-worker2.toml @@ -0,0 +1,14 @@ +# Worker Configuration. + +server-id = 102 # we don't give dm_full_account REPLICATION SLAVE privilege here, so we must specify server-id here +source-id = "mysql-replica-02" +flavor = "" +enable-gtid = false +relay-binlog-name = "" +relay-binlog-gtid = "" + +[from] +host = "127.0.0.1" +user = "dm_full" +password = "" +port = 3307 diff --git a/tests/full_mode/data/db1.prepare.sql b/tests/full_mode/data/db1.prepare.sql new file mode 100644 index 0000000000..6348194685 --- /dev/null +++ b/tests/full_mode/data/db1.prepare.sql @@ -0,0 +1,29 @@ +drop database if exists `full_mode`; +create database `full_mode`; +use `full_mode`; +create table t1 (id int, name varchar(20)); +insert into t1 (id, name) values (1, 'arya'), (2, 'catelyn'); +insert into t1 (id, name) values (3, 'Eddard Stark'); +update t1 set name = 'Arya Stark' where id = 1; +update t1 set name = 'Catelyn Stark' where name = 'catelyn'; + +-- test multi column index with generated column +alter table t1 add column info json; +alter table t1 add column gen_id int as (info->"$.id"); +alter table t1 add index multi_col(`id`, `gen_id`); +insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); +insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); +update t1 set info = '{"id": 120}' where id = 1; +update t1 set info = '{"id": 121}' where id = 2; +update t1 set info = '{"id": 122}' where id = 3; + +-- test genColumnCache is reset after ddl +alter table t1 add column info2 varchar(40); +insert into t1 (id, name, info) values (6, 'gentest', '{"id": 125, "test cache": false}'); +alter table t1 add unique key gen_idx(`gen_id`); +update t1 set name = 'gentestxx' where gen_id = 123; + +insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}'); +update t1 set name = 'gentestxxxxxx' where gen_id = 124; +-- delete with unique key +delete from t1 where gen_id > 124; diff --git a/tests/full_mode/data/db1.prepare.user.sql b/tests/full_mode/data/db1.prepare.user.sql new file mode 100644 index 0000000000..aef124de88 --- /dev/null +++ b/tests/full_mode/data/db1.prepare.user.sql @@ -0,0 +1,7 @@ +drop user if exists 'dm_full'; +flush privileges; +create user 'dm_full'@'%' identified by ''; +grant all privileges on *.* to 'dm_full'@'%'; +revoke replication slave, replication client on *.* from 'dm_full'@'%'; +revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_full'@'%'; # privileges not supported by TiDB +flush privileges; diff --git a/tests/full_mode/data/db2.prepare.sql b/tests/full_mode/data/db2.prepare.sql new file mode 100644 index 0000000000..0eb75594ac --- /dev/null +++ b/tests/full_mode/data/db2.prepare.sql @@ -0,0 +1,5 @@ +drop database if exists `full_mode`; +create database `full_mode`; +use `full_mode`; +create table t2 (id int auto_increment, name varchar(20), primary key (`id`)); +insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); diff --git a/tests/full_mode/data/db2.prepare.user.sql b/tests/full_mode/data/db2.prepare.user.sql new file mode 100644 index 0000000000..aef124de88 --- /dev/null +++ b/tests/full_mode/data/db2.prepare.user.sql @@ -0,0 +1,7 @@ +drop user if exists 'dm_full'; +flush privileges; +create user 'dm_full'@'%' identified by ''; +grant all privileges on *.* to 'dm_full'@'%'; +revoke replication slave, replication client on *.* from 'dm_full'@'%'; +revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_full'@'%'; # privileges not supported by TiDB +flush privileges; diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh new file mode 100755 index 0000000000..16c8be64f1 --- /dev/null +++ b/tests/full_mode/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -eu + +cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $cur/../_utils/test_prepare +WORK_DIR=$TEST_DIR/$TEST_NAME + +function run() { + run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 + check_contains 'Query OK, 2 rows affected' + run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 + check_contains 'Query OK, 3 rows affected' + + run_sql_file $cur/data/db1.prepare.user.sql $MYSQL_HOST1 $MYSQL_PORT1 + check_count 'Query OK, 0 rows affected' 7 + run_sql_file $cur/data/db2.prepare.user.sql $MYSQL_HOST2 $MYSQL_PORT2 + check_count 'Query OK, 0 rows affected' 7 + + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + + # start DM task only + dmctl_start_task + + # use sync_diff_inspector to check full dump loader + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml +} + +cleanup_data full_mode +# also cleanup dm processes in case of last run failed +cleanup_process $* +run $* +cleanup_process $* + +echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>" diff --git a/tests/incremental_mode/data/db1.prepare.user.sql b/tests/incremental_mode/data/db1.prepare.user.sql new file mode 100644 index 0000000000..62d30b2525 --- /dev/null +++ b/tests/incremental_mode/data/db1.prepare.user.sql @@ -0,0 +1,7 @@ +drop user if exists 'dm_incremental'; +flush privileges; +create user 'dm_incremental'@'%' identified by ''; +grant all privileges on *.* to 'dm_incremental'@'%'; +revoke select, reload on *.* from 'dm_incremental'@'%'; +revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_incremental'@'%'; # privileges not supported by TiDB +flush privileges; diff --git a/tests/incremental_mode/data/db2.prepare.user.sql b/tests/incremental_mode/data/db2.prepare.user.sql new file mode 100644 index 0000000000..62d30b2525 --- /dev/null +++ b/tests/incremental_mode/data/db2.prepare.user.sql @@ -0,0 +1,7 @@ +drop user if exists 'dm_incremental'; +flush privileges; +create user 'dm_incremental'@'%' identified by ''; +grant all privileges on *.* to 'dm_incremental'@'%'; +revoke select, reload on *.* from 'dm_incremental'@'%'; +revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_incremental'@'%'; # privileges not supported by TiDB +flush privileges; diff --git a/tests/incremental_mode/run.sh b/tests/incremental_mode/run.sh index 356abae35f..845d2c2672 100755 --- a/tests/incremental_mode/run.sh +++ b/tests/incremental_mode/run.sh @@ -36,6 +36,21 @@ function run() { run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 # start a task in `incremental` mode + # using account with limited privileges + kill_dm_worker + run_sql_file $cur/data/db1.prepare.user.sql $MYSQL_HOST1 $MYSQL_PORT1 + check_count 'Query OK, 0 rows affected' 7 + run_sql_file $cur/data/db2.prepare.user.sql $MYSQL_HOST2 $MYSQL_PORT2 + check_count 'Query OK, 0 rows affected' 7 + cat $cur/conf/dm-worker1.toml > $WORK_DIR/dm-worker1.toml + sed -i "s/root/dm_incremental/g" $WORK_DIR/dm-worker1.toml + cat $cur/conf/dm-worker2.toml > $WORK_DIR/dm-worker2.toml + sed -i "s/root/dm_incremental/g" $WORK_DIR/dm-worker2.toml + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $WORK_DIR/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $WORK_DIR/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + cat $cur/conf/dm-task.yaml > $WORK_DIR/dm-task.yaml sed -i "s/task-mode-placeholder/incremental/g" $WORK_DIR/dm-task.yaml name1=$(grep "Log: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata|awk -F: '{print $2}'|tr -d ' ') @@ -46,7 +61,8 @@ function run() { sed -i "s/binlog-pos-placeholder-1/$pos1/g" $WORK_DIR/dm-task.yaml sed -i "s/binlog-name-placeholder-2/$name2/g" $WORK_DIR/dm-task.yaml sed -i "s/binlog-pos-placeholder-2/$pos2/g" $WORK_DIR/dm-task.yaml - dmctl_start_task $WORK_DIR/dm-task.yaml + sleep 2 + dmctl_start_task $WORK_DIR/dm-task.yaml check_sync_diff $WORK_DIR $cur/conf/diff_config.toml } diff --git a/tests/others_integration.txt b/tests/others_integration.txt index 47378bd9f5..64c0cde428 100644 --- a/tests/others_integration.txt +++ b/tests/others_integration.txt @@ -1,7 +1,8 @@ all_mode +full_mode sequence_sharding sequence_safe_mode relay_interrupt start_task initial_unit -http_apis \ No newline at end of file +http_apis