*: support create table with vector indexes and others | tidb-test=c567cde7e19a55a67960a404906cb859ca44f206

errors.toml

@@ -1561,6 +1561,11 @@ error = '''
 Global Index is needed for index '%-.192s', since the unique index is not including all partitioning columns, and GLOBAL is not given as IndexOption
 '''
 
+["ddl:9014"]
+error = '''
+TiFlash backfill index failed: TiFlash backfill index failed: %s
+'''
+
 ["domain:8027"]
 error = '''
 Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV

pkg/ddl/column.go

@@ -1094,10 +1094,10 @@ func isColumnWithIndex(colName string, indices []*model.IndexInfo) bool {
 
 func isColumnCanDropWithIndex(colName string, indices []*model.IndexInfo) error {
 	for _, indexInfo := range indices {
-		if indexInfo.Primary || len(indexInfo.Columns) > 1 {
+		if indexInfo.Primary || len(indexInfo.Columns) > 1 || indexInfo.VectorInfo != nil {
 			for _, col := range indexInfo.Columns {
 				if col.Name.L == colName {
-					return dbterror.ErrCantDropColWithIndex.GenWithStack("can't drop column %s with composite index covered or Primary Key covered now", colName)
+					return dbterror.ErrCantDropColWithIndex.GenWithStack("can't drop column %s with composite index covered or Primary Key or Vector Key covered now", colName)
 				}
 			}
 		}

pkg/ddl/create_table.go

@@ -509,6 +509,40 @@ func checkGeneratedColumn(ctx sessionctx.Context, schemaName pmodel.CIStr, table
 	return nil
 }
 
+func checkVectorIndexIfNeedTiFlashReplica(ctx sessionctx.Context, tblInfo *model.TableInfo) error {
+	var hasVectorIndex bool
+	for _, idx := range tblInfo.Indices {
+		if idx.VectorInfo != nil {
+			hasVectorIndex = true
+			break
+		}
+	}
+
+	if hasVectorIndex {
+		if tblInfo.TiFlashReplica == nil {
+			replicas, err := infoschema.GetTiFlashStoreCount(ctx)
+			if err != nil {
+				return errors.Trace(err)
+			}
+			if replicas == 0 {
+				return errors.Trace(dbterror.ErrUnsupportedAddVectorIndex.FastGenByArgs("unsupported TiFlash store count is 0"))
+			}
+
+			// Always try to set to 1 as the default replica count.
+			defaultReplicas := uint64(1)
+			tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{
+				Count:          defaultReplicas,
+				LocationLabels: make([]string, 0),
+			}
+		}
+
+		if err := checkTableTypeForVectorIndex(tblInfo); err != nil {
+			return errors.Trace(err)
+		}
+	}
+	return nil
+}
+
 // checkTableInfoValidExtra is like checkTableInfoValid, but also assumes the
 // table info comes from untrusted source and performs further checks such as
 // name length and column count.
@@ -537,6 +571,10 @@ func checkTableInfoValidExtra(ctx sessionctx.Context, tbInfo *model.TableInfo) e
 	if err := checkGlobalIndexes(ctx, tbInfo); err != nil {
 		return errors.Trace(err)
 	}
+	// A special rule on Serverless is to add TiFlash replica by default if there is a vector index.
+	if err := checkVectorIndexIfNeedTiFlashReplica(ctx, tbInfo); err != nil {
+		return errors.Trace(err)
+	}
 
 	// FIXME: perform checkConstraintNames
 	if err := checkCharsetAndCollation(tbInfo.Charset, tbInfo.Collate); err != nil {
@@ -1161,14 +1199,13 @@ func BuildTableInfo(
 	}
 	foreignKeyID := tbInfo.MaxForeignKeyID
 	for _, constr := range constraints {
-		if constr.Tp == ast.ConstraintVector {
-			return nil, dbterror.ErrUnsupportedAddVectorIndex.FastGenByArgs("not currently supported")
-		}
-
-		// Build hidden columns if necessary.
-		hiddenCols, err := buildHiddenColumnInfoWithCheck(ctx, constr.Keys, pmodel.NewCIStr(constr.Name), tbInfo, tblColumns)
-		if err != nil {
-			return nil, err
+		var hiddenCols []*model.ColumnInfo
+		if constr.Tp != ast.ConstraintVector {
+			// Build hidden columns if necessary.
+			hiddenCols, err = buildHiddenColumnInfoWithCheck(ctx, constr.Keys, pmodel.NewCIStr(constr.Name), tbInfo, tblColumns)
+			if err != nil {
+				return nil, err
+			}
 		}
 		for _, hiddenCol := range hiddenCols {
 			hiddenCol.State = model.StatePublic
@@ -1233,18 +1270,23 @@ func BuildTableInfo(
 		}
 
 		var (
-			indexName       = constr.Name
-			primary, unique bool
+			indexName               = constr.Name
+			primary, unique, vector bool
 		)
 
-		// Check if the index is primary or unique.
+		// Check if the index is primary, unique or vector.
 		switch constr.Tp {
 		case ast.ConstraintPrimaryKey:
 			primary = true
 			unique = true
 			indexName = mysql.PrimaryKeyName
 		case ast.ConstraintUniq, ast.ConstraintUniqKey, ast.ConstraintUniqIndex:
 			unique = true
+		case ast.ConstraintVector:
+			if constr.Option.Visibility == ast.IndexVisibilityInvisible {
+				return nil, dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("set vector index invisible")
+			}
+			vector = true
 		}
 
 		// check constraint
@@ -1317,7 +1359,7 @@ func BuildTableInfo(
 			pmodel.NewCIStr(indexName),
 			primary,
 			unique,
-			false,
+			vector,
 			constr.Keys,
 			constr.Option,
 			model.StatePublic,

pkg/ddl/db_change_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/pkg/ddl"
 	"github.com/pingcap/tidb/pkg/domain"
+	"github.com/pingcap/tidb/pkg/domain/infosync"
 	"github.com/pingcap/tidb/pkg/executor"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta/model"
@@ -1157,6 +1158,35 @@ func TestParallelAlterAddIndex(t *testing.T) {
 	testControlParallelExecSQL(t, tk, store, dom, "", sql1, sql2, f)
 }
 
+func TestParallelAlterAddVectorIndex(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, tiflashReplicaLease, withMockTiFlash(2))
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
+	tk.MustExec("use test_db_state")
+	tk.MustExec("create table tt (a int, b vector, c vector(3), d vector(4));")
+	tk.MustExec("alter table tt set tiflash replica 2 location labels 'a','b';")
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/MockCheckVectorIndexProcess", `return(1)`))
+	defer func() {
+		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/"+
+			"ddl/MockCheckVectorIndexProcess"))
+	}()
+	tiflash := infosync.NewMockTiFlash()
+	infosync.SetMockTiFlash(tiflash)
+	defer func() {
+		tiflash.Lock()
+		tiflash.StatusServer.Close()
+		tiflash.Unlock()
+	}()
+	sql1 := "alter table tt add vector index vecIdx((vec_cosine_distance(c))) USING HNSW;"
+	sql2 := "alter table tt add vector index vecIdx1((vec_cosine_distance(c))) USING HNSW;"
+	f := func(err1, err2 error) {
+		require.NoError(t, err1)
+		require.EqualError(t, err2,
+			"[ddl:1061]DDL job rollback, error msg: vector index vecIdx function vec_cosine_distance already exist on column c")
+	}
+	testControlParallelExecSQL(t, tk, store, dom, "", sql1, sql2, f)
+}
+
 func TestParallelAlterAddExpressionIndex(t *testing.T) {
 	store, dom := testkit.CreateMockStoreAndDomain(t)
 	tk := testkit.NewTestKit(t, store)

pkg/ddl/db_integration_test.go

@@ -1967,11 +1967,13 @@ func TestDropColumnWithCompositeIndex(t *testing.T) {
 	defer tk.MustExec("drop table if exists t_drop_column_with_comp_idx")
 	tk.MustExec("create index idx_bc on t_drop_column_with_comp_idx(b, c)")
 	tk.MustExec("create index idx_b on t_drop_column_with_comp_idx(b)")
-	tk.MustGetErrMsg("alter table t_drop_column_with_comp_idx drop column b", "[ddl:8200]can't drop column b with composite index covered or Primary Key covered now")
+	tk.MustGetErrMsg("alter table t_drop_column_with_comp_idx drop column b",
+		"[ddl:8200]can't drop column b with composite index covered or Primary Key or Vector Key covered now")
 	tk.MustQuery(query).Check(testkit.Rows("idx_b YES", "idx_bc YES"))
 	tk.MustExec("alter table t_drop_column_with_comp_idx alter index idx_bc invisible")
 	tk.MustExec("alter table t_drop_column_with_comp_idx alter index idx_b invisible")
-	tk.MustGetErrMsg("alter table t_drop_column_with_comp_idx drop column b", "[ddl:8200]can't drop column b with composite index covered or Primary Key covered now")
+	tk.MustGetErrMsg("alter table t_drop_column_with_comp_idx drop column b",
+		"[ddl:8200]can't drop column b with composite index covered or Primary Key or Vector Key covered now")
 	tk.MustQuery(query).Check(testkit.Rows("idx_b NO", "idx_bc NO"))
 }
 

pkg/ddl/index.go

@@ -410,7 +410,7 @@ func buildVectorInfoWithCheck(indexPartSpecifications []*ast.IndexPartSpecificat
 	}
 	colInfo := findColumnByName(colExpr.Name.Name.L, tblInfo)
 	if colInfo == nil {
-		return nil, "", infoschema.ErrColumnNotExists.GenWithStackByArgs(colExpr.Name.Name.String())
+		return nil, "", infoschema.ErrColumnNotExists.GenWithStackByArgs(colExpr.Name.Name, tblInfo.Name)
 	}
 
 	// check duplicated function on the same column
@@ -422,7 +422,9 @@ func buildVectorInfoWithCheck(indexPartSpecifications []*ast.IndexPartSpecificat
 			continue
 		}
 		if idx.VectorInfo.DistanceMetric == distanceMetric {
-			return nil, "", dbterror.ErrDupKeyName.FastGen(fmt.Sprintf("Duplicate vector index function name 'vector index: %s, column name: %s, duplicate function name: %s'", idx.Name, colInfo.Name, f.FnName))
+			return nil, "", dbterror.ErrDupKeyName.GenWithStack(
+				fmt.Sprintf("vector index %s function %s already exist on column %s",
+					idx.Name, f.FnName, colInfo.Name))
 		}
 	}
 	if colInfo.FieldType.GetFlen() <= 0 {
@@ -543,6 +545,9 @@ func validateAlterIndexVisibility(ctx sessionctx.Context, indexName pmodel.CIStr
 			return true, nil
 		}
 	}
+	if idx.VectorInfo != nil {
+		return false, dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("set vector index invisible")
+	}
 	return false, nil
 }
 
@@ -879,11 +884,12 @@ func (w *worker) checkVectorIndexProcessOnTiFlash(jobCtx *jobContext, t *meta.Me
 
 func (w *worker) checkVectorIndexProcess(jobCtx *jobContext, tbl table.Table, job *model.Job, index *model.IndexInfo) error {
 	waitTimeout := ReorgWaitTimeout
+	ticker := time.Tick(waitTimeout)
 	for {
 		select {
 		case <-w.ddlCtx.ctx.Done():
 			return dbterror.ErrInvalidWorker.GenWithStack("worker is closed")
-		case <-time.After(waitTimeout):
+		case <-ticker:
 			logutil.DDLLogger().Info("[ddl] index backfill state running, check vector index process",
 				zap.Stringer("job", job), zap.Stringer("index name", index.Name), zap.Int64("index ID", index.ID),
 				zap.Duration("wait time", waitTimeout), zap.Int64("total added row count", job.RowCount))
@@ -916,27 +922,36 @@ func (w *worker) checkVectorIndexProcess(jobCtx *jobContext, tbl table.Table, jo
 func (w *worker) checkVectorIndexProcessOnce(jobCtx *jobContext, tbl table.Table, indexID int64) (bool, int64, error) {
 	failpoint.Inject("MockCheckVectorIndexProcess", func(val failpoint.Value) {
 		if valInt, ok := val.(int); ok {
-			if valInt == -1 {
+			if valInt < 0 {
+				failpoint.Return(false, 0, dbterror.ErrTiFlashBackfillIndex.FastGenByArgs("mock a check error"))
+			} else if valInt == 0 {
 				failpoint.Return(false, 0, nil)
 			} else {
 				failpoint.Return(true, int64(valInt), nil)
 			}
 		}
 	})
 
-	// TODO: We need to add error_msg for to show error information.
-	sql := fmt.Sprintf("select rows_stable_not_indexed, rows_stable_indexed from information_schema.tiflash_indexes where table_id = %d and index_id = %d;",
+	sql := fmt.Sprintf("select rows_stable_not_indexed, rows_stable_indexed, error_message from information_schema.tiflash_indexes where table_id = %d and index_id = %d;",
 		tbl.Meta().ID, indexID)
 	rows, err := w.sess.Execute(jobCtx.ctx, sql, "add_vector_index_check_result")
 	if err != nil || len(rows) == 0 {
 		return false, 0, errors.Trace(err)
 	}
 
-	// handle info from multiple TiFlash nodes
-	notAddedIndexCnt, addedIndexCnt := int64(0), int64(0)
+	// Get and process info from multiple TiFlash nodes.
+	notAddedIndexCnt, addedIndexCnt, errMsg := int64(0), int64(0), ""
 	for _, row := range rows {
 		notAddedIndexCnt += row.GetInt64(0)
 		addedIndexCnt += row.GetInt64(1)
+		errMsg = row.GetString(2)
+		if len(errMsg) != 0 {
+			err = dbterror.ErrTiFlashBackfillIndex.FastGenByArgs(errMsg)
+			break
+		}
+	}
+	if err != nil {
+		return false, 0, errors.Trace(err)
 	}
 	if notAddedIndexCnt != 0 {
 		return false, 0, nil