*: fix utf8 charset upgrade compatibility

config/config.go

@@ -63,18 +63,19 @@ type Config struct {
 	// TODO: We actually only support mode 2, which keeps the original case, but the comparison is case-insensitive.
 	LowerCaseTableNames int `toml:"lower-case-table-names" json:"lower-case-table-names"`
 
-	Log                 Log               `toml:"log" json:"log"`
-	Security            Security          `toml:"security" json:"security"`
-	Status              Status            `toml:"status" json:"status"`
-	Performance         Performance       `toml:"performance" json:"performance"`
-	PreparedPlanCache   PreparedPlanCache `toml:"prepared-plan-cache" json:"prepared-plan-cache"`
-	OpenTracing         OpenTracing       `toml:"opentracing" json:"opentracing"`
-	ProxyProtocol       ProxyProtocol     `toml:"proxy-protocol" json:"proxy-protocol"`
-	TiKVClient          TiKVClient        `toml:"tikv-client" json:"tikv-client"`
-	Binlog              Binlog            `toml:"binlog" json:"binlog"`
-	CompatibleKillQuery bool              `toml:"compatible-kill-query" json:"compatible-kill-query"`
-	Plugin              Plugin            `toml:"plugin" json:"plugin"`
-	CheckMb4ValueInUtf8 bool              `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	Log                     Log               `toml:"log" json:"log"`
+	Security                Security          `toml:"security" json:"security"`
+	Status                  Status            `toml:"status" json:"status"`
+	Performance             Performance       `toml:"performance" json:"performance"`
+	PreparedPlanCache       PreparedPlanCache `toml:"prepared-plan-cache" json:"prepared-plan-cache"`
+	OpenTracing             OpenTracing       `toml:"opentracing" json:"opentracing"`
+	ProxyProtocol           ProxyProtocol     `toml:"proxy-protocol" json:"proxy-protocol"`
+	TiKVClient              TiKVClient        `toml:"tikv-client" json:"tikv-client"`
+	Binlog                  Binlog            `toml:"binlog" json:"binlog"`
+	CompatibleKillQuery     bool              `toml:"compatible-kill-query" json:"compatible-kill-query"`
+	Plugin                  Plugin            `toml:"plugin" json:"plugin"`
+	CheckMb4ValueInUtf8     bool              `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	IgnoreColumnUTF8Charset bool              `toml:"ignore-column-utf8-charset" json:"ignore-column-utf8-charset"`
 }
 
 // Log is the log section of config.
@@ -267,20 +268,21 @@ type Plugin struct {
 }
 
 var defaultConf = Config{
-	Host:                "0.0.0.0",
-	AdvertiseAddress:    "",
-	Port:                4000,
-	Cors:                "",
-	Store:               "mocktikv",
-	Path:                "/tmp/tidb",
-	RunDDL:              true,
-	SplitTable:          true,
-	Lease:               "45s",
-	TokenLimit:          1000,
-	OOMAction:           "log",
-	MemQuotaQuery:       32 << 30,
-	EnableStreaming:     false,
-	CheckMb4ValueInUtf8: true,
+	Host:                    "0.0.0.0",
+	AdvertiseAddress:        "",
+	Port:                    4000,
+	Cors:                    "",
+	Store:                   "mocktikv",
+	Path:                    "/tmp/tidb",
+	RunDDL:                  true,
+	SplitTable:              true,
+	Lease:                   "45s",
+	TokenLimit:              1000,
+	OOMAction:               "log",
+	MemQuotaQuery:           32 << 30,
+	EnableStreaming:         false,
+	CheckMb4ValueInUtf8:     true,
+	IgnoreColumnUTF8Charset: true,
 	TxnLocalLatches: TxnLocalLatches{
 		Enabled:  true,
 		Capacity: 2048000,

config/config.toml.example

@@ -51,6 +51,9 @@ compatible-kill-query = false
 # check mb4 value in utf8 is used to control whether to check the mb4 characters when the charset is utf8.
 check-mb4-value-in-utf8 = true
 
+# ignore-column-utf8-charset use for upgrade compatibility. Set to true will ignore column charset and use table charset when column charset is UTF8.
+ignore-column-utf8-charset = true
+
 [log]
 # Log level: debug, info, warn, error, fatal.
 level = "info"

ddl/db_integration_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/parser/mysql"
 	tmysql "github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/infoschema"
@@ -1221,8 +1222,7 @@ func (s *testIntegrationSuite) assertAlterErrorExec(c *C, sql string) {
 func (s *testIntegrationSuite) TestAlterAlgorithm(c *C) {
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("use test")
-	s.tk.MustExec("drop table if exists t")
-	s.tk.MustExec("drop table if exists t1")
+	s.tk.MustExec("drop table if exists t, t1")
 	defer s.tk.MustExec("drop table if exists t")
 
 	s.tk.MustExec(`create table t(
@@ -1282,3 +1282,28 @@ func (s *testIntegrationSuite) TestAlterAlgorithm(c *C) {
 	s.assertAlterErrorExec(c, "alter table t default charset = utf8mb4, ALGORITHM=INPLACE")
 	s.tk.MustExec("alter table t default charset = utf8mb4, ALGORITHM=INSTANT")
 }
+
+func (s *testIntegrationSuite) TestIgnoreColumnUTF8Charset(c *C) {
+	s.tk = testkit.NewTestKit(c, s.store)
+	s.tk.MustExec("use test")
+	s.tk.MustExec("drop table if exists t")
+	defer s.tk.MustExec("drop table if exists t")
+
+	s.tk.MustExec("create table t (a varchar(10) character set utf8, b varchar(10) character set ascii) charset=utf8mb4;")
+	c.Assert(config.GetGlobalConfig().IgnoreColumnUTF8Charset, IsTrue)
+	c.Assert(config.GetGlobalConfig().CheckMb4ValueInUtf8, IsTrue)
+	s.tk.MustExec("insert into t set a= x'f09f8c80'")
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	config.GetGlobalConfig().IgnoreColumnUTF8Charset = false
+	assertErrorCode(c, s.tk, "insert into t set a= x'f09f8c80';", mysql.ErrTruncatedWrongValueForField)
+	s.tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+		"  `a` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,\n" +
+		"  `b` varchar(10) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+	config.GetGlobalConfig().IgnoreColumnUTF8Charset = true
+}

ddl/db_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/pingcap/parser/mysql"
 	tmysql "github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl"
 	testddlutil "github.com/pingcap/tidb/ddl/testutil"
 	"github.com/pingcap/tidb/domain"
@@ -2651,6 +2652,7 @@ func (s *testDBSuite) TestModifyColumnCharset(c *C) {
 	s.tk.MustExec("create table t_mcc(a varchar(8) charset utf8, b varchar(8) charset utf8)")
 	defer s.mustExec(c, "drop table t_mcc;")
 
+	config.GetGlobalConfig().IgnoreColumnUTF8Charset = false
 	result := s.tk.MustQuery(`show create table t_mcc`)
 	result.Check(testkit.Rows(
 		"t_mcc CREATE TABLE `t_mcc` (\n" +

executor/show.go

@@ -629,7 +629,9 @@ func (e *ShowExec) fetchShowCreateTable() error {
 		fmt.Fprintf(&buf, "  %s %s", escape(col.Name, sqlMode), col.GetTypeDesc())
 		if col.Charset != "binary" {
 			if col.Charset != tblCharset || col.Collate != tblCollate {
-				fmt.Fprintf(&buf, " CHARACTER SET %s COLLATE %s", col.Charset, col.Collate)
+				if col.Charset != charset.CharsetUTF8 || !config.GetGlobalConfig().IgnoreColumnUTF8Charset {
+					fmt.Fprintf(&buf, " CHARACTER SET %s COLLATE %s", col.Charset, col.Collate)
+				}
 			}
 		}
 		if col.IsGenerated() {

executor/statement_context_test.go

@@ -88,6 +88,7 @@ func (s *testSuite1) TestStatementContext(c *C) {
 	tk.MustExec("drop table if exists t1")
 	tk.MustExec("create table t1(a varchar(100) charset utf8);")
 	defer tk.MustExec("drop table if exists t1")
+	config.GetGlobalConfig().IgnoreColumnUTF8Charset = false
 	tk.MustExec("insert t1 values (unhex('f09f8c80'))")
 	c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Greater, uint16(0))
 	tk.MustQuery("select * from t1").Check(testkit.Rows(""))

server/http_handler.go

@@ -743,6 +743,17 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 				return
 			}
 		}
+		if ignoreColumnUTF8Charset := req.Form.Get("ignore-column-utf8-charset"); ignoreColumnUTF8Charset != "" {
+			switch ignoreColumnUTF8Charset {
+			case "0":
+				config.GetGlobalConfig().IgnoreColumnUTF8Charset = false
+			case "1":
+				config.GetGlobalConfig().IgnoreColumnUTF8Charset = true
+			default:
+				writeError(w, errors.New("illegal argument"))
+				return
+			}
+		}
 	} else {
 		writeData(w, config.GetGlobalConfig())
 	}

server/http_handler_test.go

@@ -663,6 +663,7 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) {
 	c.Assert(config.GetGlobalConfig().CheckMb4ValueInUtf8, Equals, true)
 	txn1, err := dbt.db.Begin()
 	c.Assert(err, IsNil)
+	config.GetGlobalConfig().IgnoreColumnUTF8Charset = false
 	_, err = txn1.Exec("insert t2 values (unhex('F0A48BAE'));")
 	c.Assert(err, NotNil)
 	txn1.Commit()
@@ -675,6 +676,20 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) {
 	c.Assert(resp.StatusCode, Equals, http.StatusOK)
 	c.Assert(config.GetGlobalConfig().CheckMb4ValueInUtf8, Equals, false)
 	dbt.mustExec("insert t2 values (unhex('f09f8c80'));")
+
+	// Test enable ignore-column-utf8-charset.
+	form.Set("ignore-column-utf8-charset", "0")
+	resp, err = http.PostForm("http://127.0.0.1:10090/settings", form)
+	c.Assert(err, IsNil)
+	c.Assert(resp.StatusCode, Equals, http.StatusOK)
+	c.Assert(config.GetGlobalConfig().IgnoreColumnUTF8Charset, Equals, false)
+
+	// Test Disable ignore-column-utf8-charset.
+	form.Set("ignore-column-utf8-charset", "1")
+	resp, err = http.PostForm("http://127.0.0.1:10090/settings", form)
+	c.Assert(err, IsNil)
+	c.Assert(resp.StatusCode, Equals, http.StatusOK)
+	c.Assert(config.GetGlobalConfig().IgnoreColumnUTF8Charset, Equals, true)
 }
 
 func (ts *HTTPHandlerTestSuite) TestPprof(c *C) {

table/column.go

@@ -192,7 +192,7 @@ func CastValue(ctx sessionctx.Context, val types.Datum, col *model.ColumnInfo) (
 			}
 			casted, err = handleWrongUtf8Value(ctx, col, &casted, str, i)
 			break
-		} else if width > 3 && utf8Charset && config.GetGlobalConfig().CheckMb4ValueInUtf8 {
+		} else if width > 3 && utf8Charset && config.GetGlobalConfig().CheckMb4ValueInUtf8 && !config.GetGlobalConfig().IgnoreColumnUTF8Charset {
 			// Handle non-BMP characters.
 			casted, err = handleWrongUtf8Value(ctx, col, &casted, str, i)
 			break