From b03249e00a7366265b3a2aa0aa356705c0758faf Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Tue, 9 Apr 2019 20:21:20 +0800 Subject: [PATCH] drainer: add config ignore-tables (#520) (#526) --- cmd/drainer/drainer.toml | 12 +++++++++--- drainer/config.go | 1 + drainer/syncer.go | 2 +- tests/filter/drainer.toml | 15 ++++++++++++--- tests/filter/run.sh | 3 +++ 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmd/drainer/drainer.toml b/cmd/drainer/drainer.toml index c51f4edb7..c94c87906 100644 --- a/cmd/drainer/drainer.toml +++ b/cmd/drainer/drainer.toml @@ -29,9 +29,6 @@ pd-urls = "http://127.0.0.1:2379" # If this is not setted, it will not set any sql-mode. # sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION" -# disable sync these schema -ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" - # number of binlog events in a transaction batch txn-batch = 20 @@ -49,10 +46,14 @@ safe-mode = false # valid values are "mysql", "pb", "tidb", "flash", "kafka" db-type = "mysql" +# disable sync these schema +ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" + ##replicate-do-db priority over replicate-do-table if have same db name ##and we support regex expression , start with '~' declare use regex expression. # #replicate-do-db = ["~^b.*","s1"] + #[[syncer.replicate-do-table]] #db-name ="test" #tbl-name = "log" @@ -61,6 +62,11 @@ db-type = "mysql" #db-name ="test" #tbl-name = "~^a.*" +# disable sync these table +#[[syncer.ignore-table]] +#db-name = "test" +#tbl-name = "log" + # the downstream mysql protocol database [syncer.to] host = "127.0.0.1" diff --git a/drainer/config.go b/drainer/config.go index 3389669f5..9f963354e 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -47,6 +47,7 @@ type SyncerConfig struct { StrSQLMode *string `toml:"sql-mode" json:"sql-mode"` SQLMode mysql.SQLMode `toml:"-" json:"-"` IgnoreSchemas string `toml:"ignore-schemas" json:"ignore-schemas"` + IgnoreTables []filter.TableName `toml:"ignore-table" json:"ignore-table"` TxnBatch int `toml:"txn-batch" json:"txn-batch"` WorkerCount int `toml:"worker-count" json:"worker-count"` To *executor.DBConfig `toml:"to" json:"to"` diff --git a/drainer/syncer.go b/drainer/syncer.go index bce2dcbc9..1f6eaef9c 100644 --- a/drainer/syncer.go +++ b/drainer/syncer.go @@ -70,7 +70,7 @@ func NewSyncer(ctx context.Context, cp checkpoint.CheckPoint, cfg *SyncerConfig) syncer.positions = make(map[string]int64) syncer.causality = loader.NewCausality() syncer.lastSyncTime = time.Now() - syncer.filter = filter.NewFilter(strings.Split(cfg.IgnoreSchemas, ","), nil, cfg.DoDBs, cfg.DoTables) + syncer.filter = filter.NewFilter(strings.Split(cfg.IgnoreSchemas, ","), cfg.IgnoreTables, cfg.DoDBs, cfg.DoTables) return syncer, nil } diff --git a/tests/filter/drainer.toml b/tests/filter/drainer.toml index aa87f7d15..c1042b93c 100644 --- a/tests/filter/drainer.toml +++ b/tests/filter/drainer.toml @@ -16,9 +16,6 @@ pd-urls = "http://127.0.0.1:2379" # syncer Configuration. [syncer] -# disable sync these schema -ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" - # number of binlog events in a transaction batch txn-batch = 1 @@ -34,18 +31,30 @@ safe-mode = false # valid values are "mysql", "pb", "tidb", "flash", "kafka" db-type = "mysql" +# disable sync these schema +ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" + ##replicate-do-db priority over replicate-do-table if have same db name ##and we support regex expression , start with '~' declare use regex expression. # replicate-do-db = ["~^do_start.*","do_name"] + [[syncer.replicate-do-table]] db-name ="test" tbl-name = "do_name" +[[syncer.replicate-do-table]] +db-name ="test" +tbl-name = "do_ignore_name" + [[syncer.replicate-do-table]] db-name ="test" tbl-name = "~^do_start.*" +[[syncer.ignore-table]] +db-name = "test" +tbl-name = "do_ignore_name" + # the downstream mysql protocol database [syncer.to] host = "127.0.0.1" diff --git a/tests/filter/run.sh b/tests/filter/run.sh index e0f3f9f92..ae0219de7 100755 --- a/tests/filter/run.sh +++ b/tests/filter/run.sh @@ -29,11 +29,13 @@ run_sql "CREATE TABLE test.do_start1(id int);" run_sql "CREATE TABLE test.do_name(id int);" run_sql "CREATE TABLE test.do_not_start1(id int);" run_sql "CREATE TABLE test.do_not_name(id int);" +run_sql "CREATE TABLE test.do_ignore_name(id int);" run_sql "INSERT INTO test.do_start1(id) VALUES (1);" run_sql "INSERT INTO test.do_name(id) VALUES (1);" run_sql "INSERT INTO test.do_not_start1(id) VALUES (1);" run_sql "INSERT INTO test.do_not_name(id) VALUES (1);" +run_sql "INSERT INTO test.do_ignore_name(id) VALUES (1);" sleep 5 @@ -50,6 +52,7 @@ check_contains "Tables_in_test: do_start1" check_contains "Tables_in_test: do_name" check_not_contains "Tables_in_test: do_not_start1" check_not_contains "Tables_in_test: do_not_name" +check_not_contains "Tables_in_test: do_ignore_name" # check DML down_run_sql "SELECT count(*) FROM test.do_start1;"