*: only retry for special message with 1105 (pingcap#330)

lichunzhu · Oct 23, 2019 · 3e766c4 · 3e766c4
1 parent 2b24e8a
commit 3e766c4
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 19 deletions.
diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go
@@ -45,6 +45,12 @@ var (
 		"binlog checksum mismatch, data may be corrupted",
 		"get event err EOF",
 	}
+
+	// Retryable1105Msgs list the error messages of some retryable error with 1105 code.
+	Retryable1105Msgs = []string{
+		"Information schema is out of date",
+		"Information schema is changed",
+	}
 )
 
 // IsRetryableError tells whether this error should retry
@@ -54,8 +60,14 @@ func IsRetryableError(err error) bool {
 	if ok {
 		switch mysqlErr.Number {
 		// ER_LOCK_DEADLOCK can retry to commit while meet deadlock
-		case tmysql.ErrUnknown, gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrQueryInterrupted, tmysql.ErrWriteConflictInTiDB, tmysql.ErrTableLocked, tmysql.ErrWriteConflict:
+		case gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrQueryInterrupted, tmysql.ErrWriteConflictInTiDB, tmysql.ErrTableLocked, tmysql.ErrWriteConflict:
 			return true
+		case tmysql.ErrUnknown:
+			for _, msg := range Retryable1105Msgs {
+				if strings.Contains(mysqlErr.Message, msg) {
+					return true
+				}
+			}
 		default:
 			return false
 		}
@@ -72,18 +84,3 @@ func IsConnectionError(err error) bool {
 	}
 	return false
 }
-
-// IsRetryableErrorFastFailFilter tells whether this error should retry,
-// filtering some incompatible DDL error to achieve fast fail.
-func IsRetryableErrorFastFailFilter(err error) bool {
-	err2 := errors.Cause(err) // check the original error
-	if mysqlErr, ok := err2.(*mysql.MySQLError); ok && mysqlErr.Number == tmysql.ErrUnknown {
-		for _, msg := range UnsupportedDDLMsgs {
-			if strings.Contains(mysqlErr.Message, msg) {
-				return false
-			}
-		}
-	}
-
-	return IsRetryableError(err)
-}
diff --git a/syncer/db.go b/syncer/db.go
@@ -238,7 +238,7 @@ func (conn *DBConn) executeSQLWithIgnore(tctx *tcontext.Context, ignoreError fun
 				sqlRetriesTotal.WithLabelValues("stmt_exec", conn.cfg.Name).Add(1)
 				return true
 			}
-			if retry.IsRetryableErrorFastFailFilter(err) {
+			if retry.IsRetryableError(err) {
 				tctx.L().Warn("execute statements", zap.Int("retry", retryTime),
 					zap.String("queries", utils.TruncateInterface(queries, -1)),
 					zap.String("arguments", utils.TruncateInterface(args, -1)))

diff --git a/syncer/error_test.go b/syncer/error_test.go
@@ -40,7 +40,6 @@ func (s *testSyncerSuite) TestIsRetryableError(c *C) {
 	}{
 		{newMysqlErr(tmysql.ErrNoDB, "no baseConn error"), false},
 		{errors.New("unknown error"), false},
-		{newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), true},
 		{newMysqlErr(tmysql.ErrDBCreateExists, "baseConn already exists"), false},
 		{driver.ErrBadConn, false},
 		{newMysqlErr(gmysql.ER_LOCK_DEADLOCK, "Deadlock found when trying to get lock; try restarting transaction"), true},
@@ -49,6 +48,10 @@ func (s *testSyncerSuite) TestIsRetryableError(c *C) {
 		{newMysqlErr(tmysql.ErrTiKVServerBusy, "tikv server busy"), true},
 		{newMysqlErr(tmysql.ErrResolveLockTimeout, "resolve lock timeout"), true},
 		{newMysqlErr(tmysql.ErrRegionUnavailable, "region unavailable"), true},
+		{newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), false},
+		{newMysqlErr(tmysql.ErrUnknown, "can't drop column with index"), false},
+		{newMysqlErr(tmysql.ErrUnknown, "Information schema is out of date"), true},
+		{newMysqlErr(tmysql.ErrUnknown, "Information schema is changed"), true},
 	}
 
 	for _, t := range cases {

diff --git a/tests/load_interrupt/run.sh b/tests/load_interrupt/run.sh
@@ -60,7 +60,7 @@ function run() {
     check_row_count 2
 
     # only failed at the first two time, will retry later and success
-    export GO_FAILPOINTS='github.com/pingcap/dm/loader/LoadExecCreateTableFailed=3*return("1105")'
+    export GO_FAILPOINTS='github.com/pingcap/dm/loader/LoadExecCreateTableFailed=3*return("1213")' # ER_LOCK_DEADLOCK, retryable error code
     run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
     run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
     check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT