*: fix utf8 charset upgrade compatibility

config/config.go

@@ -63,18 +63,19 @@ type Config struct {
 	// TODO: We actually only support mode 2, which keeps the original case, but the comparison is case-insensitive.
 	LowerCaseTableNames int `toml:"lower-case-table-names" json:"lower-case-table-names"`
 
-	Log                 Log               `toml:"log" json:"log"`
-	Security            Security          `toml:"security" json:"security"`
-	Status              Status            `toml:"status" json:"status"`
-	Performance         Performance       `toml:"performance" json:"performance"`
-	PreparedPlanCache   PreparedPlanCache `toml:"prepared-plan-cache" json:"prepared-plan-cache"`
-	OpenTracing         OpenTracing       `toml:"opentracing" json:"opentracing"`
-	ProxyProtocol       ProxyProtocol     `toml:"proxy-protocol" json:"proxy-protocol"`
-	TiKVClient          TiKVClient        `toml:"tikv-client" json:"tikv-client"`
-	Binlog              Binlog            `toml:"binlog" json:"binlog"`
-	CompatibleKillQuery bool              `toml:"compatible-kill-query" json:"compatible-kill-query"`
-	Plugin              Plugin            `toml:"plugin" json:"plugin"`
-	CheckMb4ValueInUtf8 bool              `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	Log                          Log               `toml:"log" json:"log"`
+	Security                     Security          `toml:"security" json:"security"`
+	Status                       Status            `toml:"status" json:"status"`
+	Performance                  Performance       `toml:"performance" json:"performance"`
+	PreparedPlanCache            PreparedPlanCache `toml:"prepared-plan-cache" json:"prepared-plan-cache"`
+	OpenTracing                  OpenTracing       `toml:"opentracing" json:"opentracing"`
+	ProxyProtocol                ProxyProtocol     `toml:"proxy-protocol" json:"proxy-protocol"`
+	TiKVClient                   TiKVClient        `toml:"tikv-client" json:"tikv-client"`
+	Binlog                       Binlog            `toml:"binlog" json:"binlog"`
+	CompatibleKillQuery          bool              `toml:"compatible-kill-query" json:"compatible-kill-query"`
+	Plugin                       Plugin            `toml:"plugin" json:"plugin"`
+	CheckMb4ValueInUtf8          bool              `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	TreadOldVersionUTF8AsUTF8MB4 bool              `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"`
 }
 
 // Log is the log section of config.
@@ -267,20 +268,21 @@ type Plugin struct {
 }
 
 var defaultConf = Config{
-	Host:                "0.0.0.0",
-	AdvertiseAddress:    "",
-	Port:                4000,
-	Cors:                "",
-	Store:               "mocktikv",
-	Path:                "/tmp/tidb",
-	RunDDL:              true,
-	SplitTable:          true,
-	Lease:               "45s",
-	TokenLimit:          1000,
-	OOMAction:           "log",
-	MemQuotaQuery:       32 << 30,
-	EnableStreaming:     false,
-	CheckMb4ValueInUtf8: true,
+	Host:                         "0.0.0.0",
+	AdvertiseAddress:             "",
+	Port:                         4000,
+	Cors:                         "",
+	Store:                        "mocktikv",
+	Path:                         "/tmp/tidb",
+	RunDDL:                       true,
+	SplitTable:                   true,
+	Lease:                        "45s",
+	TokenLimit:                   1000,
+	OOMAction:                    "log",
+	MemQuotaQuery:                32 << 30,
+	EnableStreaming:              false,
+	CheckMb4ValueInUtf8:          true,
+	TreadOldVersionUTF8AsUTF8MB4: true,
 	TxnLocalLatches: TxnLocalLatches{
 		Enabled:  true,
 		Capacity: 2048000,

config/config.toml.example

@@ -51,6 +51,9 @@ compatible-kill-query = false
 # check mb4 value in utf8 is used to control whether to check the mb4 characters when the charset is utf8.
 check-mb4-value-in-utf8 = true
 
+# treat-old-version-utf8-as-utf8mb4 use for upgrade compatibility. Set to true will tread old version table/column UTF8 charset as UTF8MB4.
+treat-old-version-utf8-as-utf8mb4 = true
+
 [log]
 # Log level: debug, info, warn, error, fatal.
 level = "info"

ddl/column.go

@@ -576,10 +576,6 @@ func generateOriginDefaultValue(col *model.ColumnInfo) (interface{}, error) {
 	if odValue == strings.ToUpper(ast.CurrentTimestamp) {
 		if col.Tp == mysql.TypeTimestamp {
 			odValue = time.Now().UTC().Format(types.TimeFormat)
-			// Version = 1: For OriginDefaultValue and DefaultValue of timestamp column will stores the default time in UTC time zone.
-			//              This will fix bug in version 0.
-			// TODO: remove this version field after there is no old version 0.
-			col.Version = model.ColumnInfoVersion1
 		} else if col.Tp == mysql.TypeDatetime {
 			odValue = time.Now().Format(types.TimeFormat)
 		}

ddl/db_integration_test.go

@@ -24,10 +24,12 @@ import (
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/ast"
+	"github.com/pingcap/parser/charset"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	tmysql "github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/infoschema"
@@ -40,6 +42,7 @@ import (
 	"github.com/pingcap/tidb/store/mockstore/mocktikv"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/mock"
 	"github.com/pingcap/tidb/util/testkit"
 )
 
@@ -1221,8 +1224,7 @@ func (s *testIntegrationSuite) assertAlterErrorExec(c *C, sql string) {
 func (s *testIntegrationSuite) TestAlterAlgorithm(c *C) {
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("use test")
-	s.tk.MustExec("drop table if exists t")
-	s.tk.MustExec("drop table if exists t1")
+	s.tk.MustExec("drop table if exists t, t1")
 	defer s.tk.MustExec("drop table if exists t")
 
 	s.tk.MustExec(`create table t(
@@ -1282,3 +1284,107 @@ func (s *testIntegrationSuite) TestAlterAlgorithm(c *C) {
 	s.assertAlterErrorExec(c, "alter table t default charset = utf8mb4, ALGORITHM=INPLACE")
 	s.tk.MustExec("alter table t default charset = utf8mb4, ALGORITHM=INSTANT")
 }
+
+func (s *testIntegrationSuite) TestIgnoreColumnUTF8Charset(c *C) {
+	s.tk = testkit.NewTestKit(c, s.store)
+	s.tk.MustExec("use test")
+	s.tk.MustExec("drop table if exists t")
+	defer s.tk.MustExec("drop table if exists t")
+
+	s.tk.MustExec("create table t (a varchar(10) character set utf8, b varchar(10) character set ascii) charset=utf8mb4;")
+	assertErrorCode(c, s.tk, "insert into t set a= x'f09f8c80';", mysql.ErrTruncatedWrongValueForField)
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	// Mock old version table info with column charset is utf8.
+	mockCtx := mock.NewContext()
+	mockCtx.Store = s.store
+	err := mockCtx.NewTxn(context.Background())
+	c.Assert(err, IsNil)
+	txn, err := mockCtx.Txn(true)
+	c.Assert(err, IsNil)
+	mt := meta.NewMeta(txn)
+	db, ok := domain.GetDomain(s.ctx).InfoSchema().SchemaByName(model.NewCIStr("test"))
+	c.Assert(ok, IsTrue)
+	tbl := testGetTableByName(c, s.ctx, "test", "t")
+	tblInfo := tbl.Meta().Clone()
+	tblInfo.Version = model.TableInfoVersion0
+	tblInfo.Columns[0].Version = model.ColumnInfoVersion0
+	err = mt.UpdateTable(db.ID, tblInfo)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+
+	s.tk.MustExec("alter table t add column c varchar(10) character set utf8;") //  load latest schema.
+	c.Assert(config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4, IsTrue)
+	s.tk.MustExec("insert into t set a= x'f09f8c80'")
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,\n" +
+		"  `c` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = false
+	assertErrorCode(c, s.tk, "insert into t set a= x'f09f8c80'", mysql.ErrTruncatedWrongValueForField)
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,\n" +
+		"  `c` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	// Mock old version table info with table and column charset is utf8.
+	err = mockCtx.NewTxn(context.Background())
+	c.Assert(err, IsNil)
+	txn, err = mockCtx.Txn(true)
+	c.Assert(err, IsNil)
+	mt = meta.NewMeta(txn)
+	db, ok = domain.GetDomain(s.ctx).InfoSchema().SchemaByName(model.NewCIStr("test"))
+	c.Assert(ok, IsTrue)
+	tbl = testGetTableByName(c, s.ctx, "test", "t")
+	tblInfo = tbl.Meta().Clone()
+	tblInfo.Charset = charset.CharsetUTF8
+	tblInfo.Collate = charset.CollationUTF8
+	tblInfo.Version = model.TableInfoVersion0
+	tblInfo.Columns[0].Version = model.ColumnInfoVersion0
+	err = mt.UpdateTable(db.ID, tblInfo)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+
+	s.tk.MustExec("alter table t drop column c;") //  load latest schema.
+	config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = true
+	s.tk.MustExec("insert into t set a= x'f09f8c80'")
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = false
+	assertErrorCode(c, s.tk, "insert into t set a= x'f09f8c80'", mysql.ErrTruncatedWrongValueForField)
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"))
+
+	// Test modify column charset.
+	config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = true
+	s.tk.MustExec("alter table t modify column a varchar(10) character set utf8mb4") //  change column charset.
+	tbl = testGetTableByName(c, s.ctx, "test", "t")
+	c.Assert(tbl.Meta().Columns[0].Charset, Equals, charset.CharsetUTF8MB4)
+	c.Assert(tbl.Meta().Columns[0].Collate, Equals, charset.CollationUTF8MB4)
+	c.Assert(tbl.Meta().Columns[0].Version, Equals, model.ColumnInfoVersion0)
+	s.tk.MustExec("insert into t set a= x'f09f8c80'")
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	// Test for change column should not modify the column version.
+	s.tk.MustExec("alter table t change column a a varchar(20)") //  change column .
+	tbl = testGetTableByName(c, s.ctx, "test", "t")
+	c.Assert(tbl.Meta().Columns[0].Charset, Equals, charset.CharsetUTF8MB4)
+	c.Assert(tbl.Meta().Columns[0].Collate, Equals, charset.CollationUTF8MB4)
+	c.Assert(tbl.Meta().Columns[0].Version, Equals, model.ColumnInfoVersion0)
+}

ddl/ddl_api.go

@@ -336,10 +336,6 @@ func convertTimestampDefaultValToUTC(ctx sessionctx.Context, defaultVal interfac
 				return defaultVal, errors.Trace(err)
 			}
 			defaultVal = t.String()
-			// Version = 1: For OriginDefaultValue and DefaultValue of timestamp column will stores the default time in UTC time zone.
-			//              This will fix bug in version 0.
-			// TODO: remove this version field after there is no old version 0.
-			col.Version = model.ColumnInfoVersion1
 		}
 	}
 	return defaultVal, nil
@@ -361,6 +357,10 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
 		Offset:    offset,
 		Name:      colDef.Name.Name,
 		FieldType: *colDef.Tp,
+		// Version = 1: For OriginDefaultValue and DefaultValue of timestamp column will stores the default time in UTC time zone.
+		//              This will fix bug in version 0.
+		// TODO: remove this version field after there is no old version 0.
+		Version: model.ColumnInfoVersion1,
 	})
 
 	if !isExplicitTimeStamp() {
@@ -1294,19 +1294,21 @@ func buildViewInfoWithTableColumns(ctx sessionctx.Context, s *ast.CreateViewStmt
 	if s.Cols == nil {
 		for i, v := range schemaCols {
 			tableColumns[i] = table.ToColumn(&model.ColumnInfo{
-				Name:   v.AsName,
-				ID:     int64(i),
-				Offset: i,
-				State:  model.StatePublic,
+				Name:    v.AsName,
+				ID:      int64(i),
+				Offset:  i,
+				State:   model.StatePublic,
+				Version: model.ColumnInfoVersion1,
 			})
 		}
 	} else {
 		for i, v := range s.Cols {
 			tableColumns[i] = table.ToColumn(&model.ColumnInfo{
-				Name:   v,
-				ID:     int64(i),
-				Offset: i,
-				State:  model.StatePublic,
+				Name:    v,
+				ID:      int64(i),
+				Offset:  i,
+				State:   model.StatePublic,
+				Version: model.ColumnInfoVersion1,
 			})
 		}
 	}
@@ -2208,6 +2210,7 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or
 		OriginDefaultValue: col.OriginDefaultValue,
 		FieldType:          *specNewColumn.Tp,
 		Name:               newColName,
+		Version:            col.Version,
 	})
 
 	// TODO: Remove it when all table versions are greater than or equal to TableInfoVersion1.

executor/show.go

@@ -621,15 +621,24 @@ func (e *ShowExec) fetchShowCreateTable() error {
 	if len(tblCollate) == 0 {
 		tblCollate = getDefaultCollate(tblCharset)
 	}
+	if tb.Meta().Version == model.TableInfoVersion0 && config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 && tblCharset == charset.CharsetUTF8 {
+		tblCharset = charset.CharsetUTF8MB4
+		tblCollate = charset.CollationUTF8MB4
+	}
 
 	fmt.Fprintf(&buf, "CREATE TABLE %s (\n", escape(tb.Meta().Name, sqlMode))
 	var pkCol *table.Column
 	var hasAutoIncID bool
 	for i, col := range tb.Cols() {
 		fmt.Fprintf(&buf, "  %s %s", escape(col.Name, sqlMode), col.GetTypeDesc())
 		if col.Charset != "binary" {
-			if col.Charset != tblCharset || col.Collate != tblCollate {
-				fmt.Fprintf(&buf, " CHARACTER SET %s COLLATE %s", col.Charset, col.Collate)
+			colCharset, colCollate := col.Charset, col.Collate
+			if col.Version == model.ColumnInfoVersion0 && config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 && colCharset == charset.CharsetUTF8 {
+				colCharset = charset.CharsetUTF8MB4
+				colCollate = charset.CollationUTF8MB4
+			}
+			if colCharset != tblCharset || colCollate != tblCollate {
+				fmt.Fprintf(&buf, " CHARACTER SET %s COLLATE %s", colCharset, colCollate)
 			}
 		}
 		if col.IsGenerated() {

server/http_handler.go

@@ -743,6 +743,17 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 				return
 			}
 		}
+		if treadOldVersionUTF8AsUTF8MB4 := req.Form.Get("treat-old-version-utf8-as-utf8mb4"); treadOldVersionUTF8AsUTF8MB4 != "" {
+			switch treadOldVersionUTF8AsUTF8MB4 {
+			case "0":
+				config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = false
+			case "1":
+				config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 = true
+			default:
+				writeError(w, errors.New("illegal argument"))
+				return
+			}
+		}
 	} else {
 		writeData(w, config.GetGlobalConfig())
 	}

server/http_handler_test.go

@@ -675,6 +675,20 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) {
 	c.Assert(resp.StatusCode, Equals, http.StatusOK)
 	c.Assert(config.GetGlobalConfig().CheckMb4ValueInUtf8, Equals, false)
 	dbt.mustExec("insert t2 values (unhex('f09f8c80'));")
+
+	// Test enable treat-old-version-utf8-as-utf8mb4.
+	form.Set("treat-old-version-utf8-as-utf8mb4", "0")
+	resp, err = http.PostForm("http://127.0.0.1:10090/settings", form)
+	c.Assert(err, IsNil)
+	c.Assert(resp.StatusCode, Equals, http.StatusOK)
+	c.Assert(config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4, Equals, false)
+
+	// Test Disable treat-old-version-utf8-as-utf8mb4.
+	form.Set("treat-old-version-utf8-as-utf8mb4", "1")
+	resp, err = http.PostForm("http://127.0.0.1:10090/settings", form)
+	c.Assert(err, IsNil)
+	c.Assert(resp.StatusCode, Equals, http.StatusOK)
+	c.Assert(config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4, Equals, true)
 }
 
 func (ts *HTTPHandlerTestSuite) TestPprof(c *C) {

table/column.go

@@ -183,6 +183,12 @@ func CastValue(ctx sessionctx.Context, val types.Datum, col *model.ColumnInfo) (
 	}
 	str := casted.GetString()
 	utf8Charset := col.Charset == mysql.UTF8Charset
+	doUTF8Check := utf8Charset && config.GetGlobalConfig().CheckMb4ValueInUtf8
+	if doUTF8Check && config.GetGlobalConfig().TreadOldVersionUTF8AsUTF8MB4 && col.Version == model.ColumnInfoVersion0 {
+		// Skip utf8 check for old version column. Because old TiDB don't have this check too.
+		// This is for compatibility. We may remove this in future.
+		doUTF8Check = false
+	}
 	for i, w := 0, 0; i < len(str); i += w {
 		runeValue, width := utf8.DecodeRuneInString(str[i:])
 		if runeValue == utf8.RuneError {
@@ -192,7 +198,7 @@ func CastValue(ctx sessionctx.Context, val types.Datum, col *model.ColumnInfo) (
 			}
 			casted, err = handleWrongUtf8Value(ctx, col, &casted, str, i)
 			break
-		} else if width > 3 && utf8Charset && config.GetGlobalConfig().CheckMb4ValueInUtf8 {
+		} else if width > 3 && doUTF8Check {
 			// Handle non-BMP characters.
 			casted, err = handleWrongUtf8Value(ctx, col, &casted, str, i)
 			break