From 9784615566d565a36e5a2401d48c84f1c82dca9d Mon Sep 17 00:00:00 2001 From: Benjamin2037 Date: Wed, 23 Nov 2022 15:45:58 +0800 Subject: [PATCH] ddl: Add index fast path to be a default add index path (#39267) ref pingcap/tidb#35983 --- ddl/cancel_test.go | 2 ++ ddl/column_modify_test.go | 1 + ddl/db_change_test.go | 2 ++ ddl/db_test.go | 2 ++ ddl/failtest/fail_db_test.go | 5 +++++ ddl/index_change_test.go | 2 ++ ddl/index_merge_tmp_test.go | 1 - ddl/multi_schema_change_test.go | 3 +++ ddl/serial_test.go | 2 ++ ddl/stat_test.go | 9 ++++++--- ddl/table_modify_test.go | 1 + expression/integration_test.go | 1 + session/schema_test.go | 1 + sessionctx/variable/sysvar_test.go | 4 ++-- sessionctx/variable/tidb_vars.go | 2 +- telemetry/data_feature_usage_test.go | 8 ++++---- .../realtikvtest/pessimistictest/pessimistic_test.go | 12 ++++++++++++ 17 files changed, 47 insertions(+), 11 deletions(-) diff --git a/ddl/cancel_test.go b/ddl/cancel_test.go index cd446f3441baf..431b97357b2db 100644 --- a/ddl/cancel_test.go +++ b/ddl/cancel_test.go @@ -232,6 +232,8 @@ func TestCancel(t *testing.T) { // Prepare schema. tk.MustExec("use test") + // TODO: Will check why tidb_ddl_enable_fast_reorg could not default be on in another PR. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("drop table if exists t_partition;") tk.MustExec(`create table t_partition ( c1 int, c2 int, c3 int diff --git a/ddl/column_modify_test.go b/ddl/column_modify_test.go index f933182737d05..2055f9df9fa9c 100644 --- a/ddl/column_modify_test.go +++ b/ddl/column_modify_test.go @@ -1036,6 +1036,7 @@ func TestWriteReorgForColumnTypeChangeOnAmendTxn(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_metadata_lock=0") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0") tk.MustExec("set global tidb_enable_amend_pessimistic_txn = ON") defer tk.MustExec("set global tidb_enable_amend_pessimistic_txn = OFF") diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index d865c970e7f42..ee0634215c465 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -1719,6 +1719,8 @@ func TestCreateUniqueExpressionIndex(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("create table t(a int default 0, b int default 0)") tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3), (4, 4)") diff --git a/ddl/db_test.go b/ddl/db_test.go index 4a739520dfb81..ce2ea307e97af 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -985,6 +985,7 @@ func TestCommitTxnWithIndexChange(t *testing.T) { // Prepare work. tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_metadata_lock=0") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk.MustExec("use test") tk.MustExec("create table t1 (c1 int primary key, c2 int, c3 int, index ok2(c2))") @@ -1385,6 +1386,7 @@ func TestAmendTxnSavepointWithDDL(t *testing.T) { tk.MustExec("use test;") tk.MustExec("set global tidb_enable_metadata_lock=0") tk2.MustExec("use test;") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") prepareFn := func() { diff --git a/ddl/failtest/fail_db_test.go b/ddl/failtest/fail_db_test.go index d12c2182f9730..ece5b804ae336 100644 --- a/ddl/failtest/fail_db_test.go +++ b/ddl/failtest/fail_db_test.go @@ -320,6 +320,8 @@ func TestRunDDLJobPanicEnableClusteredIndex(t *testing.T) { s := createFailDBSuite(t) testAddIndexWorkerNum(t, s, func(tk *testkit.TestKit) { tk.Session().GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1, c3))") }) } @@ -328,6 +330,8 @@ func TestRunDDLJobPanicEnableClusteredIndex(t *testing.T) { func TestRunDDLJobPanicDisableClusteredIndex(t *testing.T) { s := createFailDBSuite(t) testAddIndexWorkerNum(t, s, func(tk *testkit.TestKit) { + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1))") }) } @@ -420,6 +424,7 @@ func TestPartitionAddIndexGC(t *testing.T) { s := createFailDBSuite(t) tk := testkit.NewTestKit(t, s.store) tk.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec(`create table partition_add_idx ( id int not null, hired date not null diff --git a/ddl/index_change_test.go b/ddl/index_change_test.go index b5d2c9d6ce983..85352cb6d08d1 100644 --- a/ddl/index_change_test.go +++ b/ddl/index_change_test.go @@ -37,6 +37,8 @@ func TestIndexChange(t *testing.T) { ddl.SetWaitTimeWhenErrorOccurred(1 * time.Microsecond) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") + // TODO: Will check why tidb_ddl_enable_fast_reorg could not default be on in another PR. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("create table t (c1 int primary key, c2 int)") tk.MustExec("insert t values (1, 1), (2, 2), (3, 3);") diff --git a/ddl/index_merge_tmp_test.go b/ddl/index_merge_tmp_test.go index 550c9b2daae5c..8dae95e590438 100644 --- a/ddl/index_merge_tmp_test.go +++ b/ddl/index_merge_tmp_test.go @@ -255,7 +255,6 @@ func TestPessimisticAmendIncompatibleWithFastReorg(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;") - tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;") tk.MustGetErrMsg("set @@tidb_enable_amend_pessimistic_txn = 1;", "amend pessimistic transactions is not compatible with tidb_ddl_enable_fast_reorg") diff --git a/ddl/multi_schema_change_test.go b/ddl/multi_schema_change_test.go index d23e88cd355a8..d48aa6752f8ee 100644 --- a/ddl/multi_schema_change_test.go +++ b/ddl/multi_schema_change_test.go @@ -568,6 +568,8 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") originHook := dom.DDL().GetHook() // Test cancel successfully. @@ -1014,6 +1016,7 @@ func TestMultiSchemaChangeMixCancelled(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("create table t (a int, b int, c int, index i1(c), index i2(c));") tk.MustExec("insert into t values (1, 2, 3);") diff --git a/ddl/serial_test.go b/ddl/serial_test.go index d99036ed4dc51..d0f2ada6521cb 100644 --- a/ddl/serial_test.go +++ b/ddl/serial_test.go @@ -432,6 +432,8 @@ func TestCancelAddIndexPanic(t *testing.T) { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/errorMockPanic")) }() tk.MustExec("use test") + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("drop table if exists t") tk.MustExec("create table t(c1 int, c2 int)") diff --git a/ddl/stat_test.go b/ddl/stat_test.go index 832e39061ce46..260a98ab25f6d 100644 --- a/ddl/stat_test.go +++ b/ddl/stat_test.go @@ -44,13 +44,16 @@ func TestDDLStatsInfo(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease) d := domain.DDL() + tk := testkit.NewTestKit(t, store) + ctx := tk.Session() dbInfo, err := testSchemaInfo(store, "test_stat") require.NoError(t, err) - testCreateSchema(t, testkit.NewTestKit(t, store).Session(), d, dbInfo) + testCreateSchema(t, ctx, d, dbInfo) tblInfo, err := testTableInfo(store, "t", 2) require.NoError(t, err) - testCreateTable(t, testkit.NewTestKit(t, store).Session(), d, dbInfo, tblInfo) - ctx := testkit.NewTestKit(t, store).Session() + testCreateTable(t, ctx, d, dbInfo, tblInfo) + // TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") err = sessiontxn.NewTxn(context.Background(), ctx) require.NoError(t, err) diff --git a/ddl/table_modify_test.go b/ddl/table_modify_test.go index f4b273771fd46..c042d266ac9e2 100644 --- a/ddl/table_modify_test.go +++ b/ddl/table_modify_test.go @@ -163,6 +163,7 @@ func TestLockTableReadOnly(t *testing.T) { tk1.MustExec("admin cleanup table lock t1") tk2.MustExec("insert into t1 set a=1, b=2") + tk1.MustExec("set global tidb_ddl_enable_fast_reorg = 0") tk1.MustExec("set tidb_enable_amend_pessimistic_txn = 1") tk1.MustExec("begin pessimistic") tk1.MustQuery("select * from t1 where a = 1").Check(testkit.Rows("1 2")) diff --git a/expression/integration_test.go b/expression/integration_test.go index 83aaea0a8675d..bd0fe73682709 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4956,6 +4956,7 @@ func TestSchemaDMLNotChange(t *testing.T) { tk2 := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("set global tidb_enable_metadata_lock=0") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk2.MustExec("use test") tk.MustExec("drop table if exists t") diff --git a/session/schema_test.go b/session/schema_test.go index f312d253418f2..3ec05f5a22530 100644 --- a/session/schema_test.go +++ b/session/schema_test.go @@ -134,6 +134,7 @@ func TestRetrySchemaChangeForEmptyChange(t *testing.T) { tk1.MustExec("commit") // TODO remove this enable after fixing table delta map. + tk1.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk1.MustExec("set tidb_enable_amend_pessimistic_txn = 1") tk1.MustExec("begin pessimistic") tk2.MustExec("alter table t add k int") diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index b7b22bb0ffa0a..3613870004bad 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -692,8 +692,8 @@ func TestSetTIDBFastDDL(t *testing.T) { vars.GlobalVarsAccessor = mock fastDDL := GetSysVar(TiDBDDLEnableFastReorg) - // Default off - require.Equal(t, fastDDL.Value, Off) + // Default true + require.Equal(t, fastDDL.Value, On) // Set to On err := mock.SetGlobalSysVar(context.Background(), TiDBDDLEnableFastReorg, On) diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 3511775de08f1..e4fa0b671cebe 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -1065,7 +1065,7 @@ const ( DefTiFlashFastScan = false DefMemoryUsageAlarmRatio = 0.7 DefMemoryUsageAlarmKeepRecordNum = 5 - DefTiDBEnableFastReorg = false + DefTiDBEnableFastReorg = true DefTiDBDDLDiskQuota = 100 * 1024 * 1024 * 1024 // 100GB DefExecutorConcurrency = 5 DefTiDBEnableGeneralPlanCache = false diff --git a/telemetry/data_feature_usage_test.go b/telemetry/data_feature_usage_test.go index f2d212f39b41b..81b1f2e6a1f01 100644 --- a/telemetry/data_feature_usage_test.go +++ b/telemetry/data_feature_usage_test.go @@ -498,7 +498,7 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) { require.NoError(t, err) allow := ddl.IsEnableFastReorg() - require.Equal(t, false, allow) + require.Equal(t, true, allow) tk.MustExec("set global tidb_enable_metadata_lock = 0") tk.MustExec("use test") tk.MustExec("drop table if exists tele_t") @@ -507,7 +507,7 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) { tk.MustExec("alter table tele_t add index idx_org(b)") usage, err = telemetry.GetFeatureUsage(tk.Session()) require.NoError(t, err) - require.Equal(t, int64(0), usage.DDLUsageCounter.AddIndexIngestUsed) + require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed) require.Equal(t, false, usage.DDLUsageCounter.MetadataLockUsed) tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = on") @@ -516,11 +516,11 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) { require.Equal(t, true, allow) usage, err = telemetry.GetFeatureUsage(tk.Session()) require.NoError(t, err) - require.Equal(t, int64(0), usage.DDLUsageCounter.AddIndexIngestUsed) + require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed) tk.MustExec("alter table tele_t add index idx_new(b)") usage, err = telemetry.GetFeatureUsage(tk.Session()) require.NoError(t, err) - require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed) + require.Equal(t, int64(2), usage.DDLUsageCounter.AddIndexIngestUsed) require.Equal(t, true, usage.DDLUsageCounter.MetadataLockUsed) } diff --git a/tests/realtikvtest/pessimistictest/pessimistic_test.go b/tests/realtikvtest/pessimistictest/pessimistic_test.go index 5e2da1d200651..0383272f8b1ed 100644 --- a/tests/realtikvtest/pessimistictest/pessimistic_test.go +++ b/tests/realtikvtest/pessimistictest/pessimistic_test.go @@ -1850,6 +1850,7 @@ func TestPessimisticTxnWithDDLAddDropColumn(t *testing.T) { // tk2 starts a pessimistic transaction and make some changes on table t1. // tk executes some ddl statements add/drop column on table t1. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk.MustExec("begin pessimistic") tk.MustExec("update t1 set c2 = c1 * 10") @@ -1880,6 +1881,7 @@ func TestPessimisticTxnWithDDLChangeColumn(t *testing.T) { tk.MustExec("insert t1 values (1, 77, 'a'), (2, 88, 'b')") // Extend column field length is acceptable. + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk.MustExec("begin pessimistic") tk.MustExec("update t1 set c2 = c1 * 10") @@ -2153,6 +2155,7 @@ func TestAmendTxnVariable(t *testing.T) { tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 0;") tk3.MustExec("begin pessimistic") tk3.MustExec("insert into t1 values(3, 3, 3)") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk.MustExec("begin pessimistic") tk.MustExec("insert into t1 values(4, 4, 4)") @@ -2173,6 +2176,7 @@ func TestAmendTxnVariable(t *testing.T) { tk4.MustExec("insert into t1 values(5, 5, 5, 5)") tk2.MustExec("alter table t1 drop column new_col") require.Error(t, tk4.ExecToErr("commit")) + tk4.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk4.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk4.MustExec("begin pessimistic") tk4.MustExec("insert into t1 values(5, 5, 5)") @@ -2303,6 +2307,7 @@ func TestAsyncCommitWithSchemaChange(t *testing.T) { tk.MustExec("insert into tk values(1, 1, 1)") tk2 := createAsyncCommitTestKit(t, store) tk3 := createAsyncCommitTestKit(t, store) + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") @@ -2377,6 +2382,7 @@ func Test1PCWithSchemaChange(t *testing.T) { tk.MustExec("drop table if exists tk") tk.MustExec("create table tk (c1 int primary key, c2 int)") tk.MustExec("insert into tk values (1, 1)") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") @@ -2424,6 +2430,7 @@ func TestAmendForUniqueIndex(t *testing.T) { tk2 := testkit.NewTestKit(t, store) tk.MustExec("use test") tk2.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk2.MustExec("drop table if exists t1") @@ -2546,6 +2553,7 @@ func TestAmendWithColumnTypeChange(t *testing.T) { tk.MustExec("set global tidb_enable_metadata_lock=0") tk2.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") tk2.MustExec("drop table if exists t") @@ -2563,6 +2571,7 @@ func TestIssue21498(t *testing.T) { tk2 := testkit.NewTestKit(t, store) tk.MustExec("use test") tk2.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1") for _, partition := range []bool{false, true} { @@ -2732,6 +2741,7 @@ func TestPlanCacheSchemaChange(t *testing.T) { tk.MustExec("create table t (id int primary key, v int, unique index iv (v), vv int)") tk.MustExec("insert into t values(1, 1, 1), (2, 2, 2), (4, 4, 4)") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0") tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1") tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1") @@ -2899,6 +2909,7 @@ func TestAmendForIndexChange(t *testing.T) { tk.MustExec("set global tidb_enable_metadata_lock=0") tk2.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = OFF;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = ON;") tk.Session().GetSessionVars().EnableAsyncCommit = false tk.Session().GetSessionVars().Enable1PC = false @@ -2974,6 +2985,7 @@ func TestAmendForColumnChange(t *testing.T) { tk.MustExec("set global tidb_enable_metadata_lock=0") tk2.MustExec("use test") + tk.MustExec("set global tidb_ddl_enable_fast_reorg = OFF;") tk.MustExec("set tidb_enable_amend_pessimistic_txn = ON;") tk2.MustExec("drop table if exists t1")