statistics: improve test to make test more fast

br/pkg/storage/BUILD.bazel

@@ -56,7 +56,7 @@ go_library(
 
 go_test(
     name = "storage_test",
-    timeout = "moderate",
+    timeout = "short",
     srcs = [
         "azblob_test.go",
         "compress_test.go",
@@ -69,6 +69,7 @@ go_test(
     ],
     embed = [":storage"],
     flaky = True,
+    shard_count = 40,
     deps = [
         "//br/pkg/mock",
         "@com_github_aws_aws_sdk_go//aws",

ddl/failtest/BUILD.bazel

@@ -2,12 +2,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_test(
     name = "failtest_test",
-    timeout = "moderate",
+    timeout = "short",
     srcs = [
         "fail_db_test.go",
         "main_test.go",
     ],
     flaky = True,
+    shard_count = 14,
     deps = [
         "//config",
         "//ddl",

ddl/indexmergetest/BUILD.bazel

@@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_test(
     name = "indexmergetest_test",
-    timeout = "moderate",
+    timeout = "short",
     srcs = [
         "main_test.go",
         "merge_test.go",
     ],
     flaky = True,
     race = "on",
-    shard_count = 4,
+    shard_count = 18,
     deps = [
         "//config",
         "//ddl",

statistics/handle/BUILD.bazel

@@ -59,17 +59,15 @@ go_library(
 
 go_test(
     name = "handle_test",
-    timeout = "moderate",
+    timeout = "short",
     srcs = [
         "ddl_test.go",
         "dump_test.go",
         "gc_test.go",
         "handle_hist_test.go",
-        "handle_test.go",
         "lru_cache_test.go",
         "main_test.go",
         "update_list_test.go",
-        "update_test.go",
     ],
     embed = [":handle"],
     flaky = True,
@@ -78,28 +76,21 @@ go_test(
     deps = [
         "//config",
         "//domain",
-        "//metrics",
         "//parser/model",
         "//parser/mysql",
-        "//session",
         "//sessionctx/stmtctx",
         "//sessionctx/variable",
         "//statistics",
+        "//statistics/handle/internal",
         "//testkit",
         "//testkit/testsetup",
         "//types",
         "//util",
-        "//util/codec",
-        "//util/collate",
-        "//util/israce",
         "//util/mathutil",
         "//util/mock",
-        "//util/ranger",
         "//util/sqlexec",
         "@com_github_pingcap_failpoint//:failpoint",
-        "@com_github_prometheus_client_model//go",
         "@com_github_stretchr_testify//require",
-        "@com_github_tikv_client_go_v2//oracle",
         "@org_uber_go_goleak//:goleak",
     ],
 )

statistics/handle/dump_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/statistics/handle"
+	"github.com/pingcap/tidb/statistics/handle/internal"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/util"
 	"github.com/stretchr/testify/require"
@@ -53,7 +54,7 @@ func requireTableEqual(t *testing.T, a *statistics.Table, b *statistics.Table) {
 		}
 		require.True(t, a.Indices[i].TopN.Equal(b.Indices[i].TopN))
 	}
-	require.True(t, isSameExtendedStats(a.ExtendedStats, b.ExtendedStats))
+	require.True(t, internal.IsSameExtendedStats(a.ExtendedStats, b.ExtendedStats))
 }
 
 func cleanStats(tk *testkit.TestKit, do *domain.Domain) {

statistics/handle/handletest/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "handletest_test",
+    timeout = "short",
+    srcs = [
+        "handle_test.go",
+        "main_test.go",
+    ],
+    flaky = True,
+    shard_count = 50,
+    deps = [
+        "//config",
+        "//domain",
+        "//parser/model",
+        "//session",
+        "//sessionctx/variable",
+        "//statistics",
+        "//statistics/handle",
+        "//statistics/handle/internal",
+        "//testkit",
+        "//testkit/testsetup",
+        "//types",
+        "//util/collate",
+        "//util/israce",
+        "//util/mock",
+        "//util/ranger",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_stretchr_testify//require",
+        "@com_github_tikv_client_go_v2//oracle",
+        "@org_uber_go_goleak//:goleak",
+    ],
+)

statistics/handle/handle_test.go → statistics/handle/handletest/handle_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package handle_test
+package handletest
 
 import (
 	"bytes"
@@ -32,6 +32,7 @@ import (
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/statistics/handle"
+	"github.com/pingcap/tidb/statistics/handle/internal"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/collate"
@@ -144,63 +145,6 @@ func TestStatsCacheMemTracker(t *testing.T) {
 	require.False(t, statsTbl.Pseudo)
 }
 
-func assertTableEqual(t *testing.T, a *statistics.Table, b *statistics.Table) {
-	require.Equal(t, b.Count, a.Count)
-	require.Equal(t, b.ModifyCount, a.ModifyCount)
-	require.Len(t, a.Columns, len(b.Columns))
-	for i := range a.Columns {
-		require.Equal(t, b.Columns[i].Count, a.Columns[i].Count)
-		require.True(t, statistics.HistogramEqual(&a.Columns[i].Histogram, &b.Columns[i].Histogram, false))
-		if a.Columns[i].CMSketch == nil {
-			require.Nil(t, b.Columns[i].CMSketch)
-		} else {
-			require.True(t, a.Columns[i].CMSketch.Equal(b.Columns[i].CMSketch))
-		}
-		// The nil case has been considered in (*TopN).Equal() so we don't need to consider it here.
-		require.Truef(t, a.Columns[i].TopN.Equal(b.Columns[i].TopN), "%v, %v", a.Columns[i].TopN, b.Columns[i].TopN)
-	}
-	require.Len(t, a.Indices, len(b.Indices))
-	for i := range a.Indices {
-		require.True(t, statistics.HistogramEqual(&a.Indices[i].Histogram, &b.Indices[i].Histogram, false))
-		if a.Indices[i].CMSketch == nil {
-			require.Nil(t, b.Indices[i].CMSketch)
-		} else {
-			require.True(t, a.Indices[i].CMSketch.Equal(b.Indices[i].CMSketch))
-		}
-		require.True(t, a.Indices[i].TopN.Equal(b.Indices[i].TopN))
-	}
-	require.True(t, isSameExtendedStats(a.ExtendedStats, b.ExtendedStats))
-}
-
-func isSameExtendedStats(a, b *statistics.ExtendedStatsColl) bool {
-	aEmpty := (a == nil) || len(a.Stats) == 0
-	bEmpty := (b == nil) || len(b.Stats) == 0
-	if (aEmpty && !bEmpty) || (!aEmpty && bEmpty) {
-		return false
-	}
-	if aEmpty && bEmpty {
-		return true
-	}
-	if len(a.Stats) != len(b.Stats) {
-		return false
-	}
-	for aKey, aItem := range a.Stats {
-		bItem, ok := b.Stats[aKey]
-		if !ok {
-			return false
-		}
-		for i, id := range aItem.ColIDs {
-			if id != bItem.ColIDs[i] {
-				return false
-			}
-		}
-		if (aItem.Tp != bItem.Tp) || (aItem.ScalarVals != bItem.ScalarVals) || (aItem.StringVals != bItem.StringVals) {
-			return false
-		}
-	}
-	return true
-}
-
 func TestStatsStoreAndLoad(t *testing.T) {
 	store, dom := testkit.CreateMockStoreAndDomain(t)
 	testKit := testkit.NewTestKit(t, store)
@@ -226,7 +170,7 @@ func TestStatsStoreAndLoad(t *testing.T) {
 	statsTbl2 := do.StatsHandle().GetTableStats(tableInfo)
 	require.False(t, statsTbl2.Pseudo)
 	require.Equal(t, int64(recordCount), statsTbl2.Count)
-	assertTableEqual(t, statsTbl1, statsTbl2)
+	internal.AssertTableEqual(t, statsTbl1, statsTbl2)
 }
 
 func TestEmptyTable(t *testing.T) {
@@ -510,7 +454,7 @@ func TestInitStats(t *testing.T) {
 	h.Clear()
 	require.NoError(t, h.Update(is))
 	table1 := h.GetTableStats(tbl.Meta())
-	assertTableEqual(t, table0, table1)
+	internal.AssertTableEqual(t, table0, table1)
 	h.SetLease(0)
 }
 
@@ -540,7 +484,7 @@ func TestInitStatsVer2(t *testing.T) {
 	h.Clear()
 	require.NoError(t, h.Update(is))
 	table1 := h.GetTableStats(tbl.Meta())
-	assertTableEqual(t, table0, table1)
+	internal.AssertTableEqual(t, table0, table1)
 	h.SetLease(0)
 }
 
@@ -2484,7 +2428,7 @@ func TestFeedbackWithGlobalStats(t *testing.T) {
 		statsTblAfter := h.GetTableStats(tblInfo)
 		// assert that statistics not changed
 		// the feedback can not work for the partition table in both static and dynamic mode
-		assertTableEqual(t, statsBefore, statsTblAfter)
+		internal.AssertTableEqual(t, statsBefore, statsTblAfter)
 	}
 
 	// Case 2: Feedback wouldn't be applied on version 2 and global-level statistics.

statistics/handle/handletest/main_test.go

@@ -0,0 +1,33 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handletest
+
+import (
+	"testing"
+
+	"github.com/pingcap/tidb/testkit/testsetup"
+	"go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+	opts := []goleak.Option{
+		goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+		goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+		goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+		goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+	}
+	testsetup.SetupForCommonTest()
+	goleak.VerifyTestMain(m, opts...)
+}

statistics/handle/internal/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "internal",
+    srcs = ["testutil.go"],
+    importpath = "github.com/pingcap/tidb/statistics/handle/internal",
+    visibility = ["//statistics/handle:__subpackages__"],
+    deps = [
+        "//statistics",
+        "@com_github_stretchr_testify//require",
+    ],
+)

statistics/handle/internal/testutil.go

@@ -0,0 +1,81 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/pingcap/tidb/statistics"
+	"github.com/stretchr/testify/require"
+)
+
+// AssertTableEqual is to assert whether two table is equal
+func AssertTableEqual(t *testing.T, a *statistics.Table, b *statistics.Table) {
+	require.Equal(t, b.Count, a.Count)
+	require.Equal(t, b.ModifyCount, a.ModifyCount)
+	require.Len(t, a.Columns, len(b.Columns))
+	for i := range a.Columns {
+		require.Equal(t, b.Columns[i].Count, a.Columns[i].Count)
+		require.True(t, statistics.HistogramEqual(&a.Columns[i].Histogram, &b.Columns[i].Histogram, false))
+		if a.Columns[i].CMSketch == nil {
+			require.Nil(t, b.Columns[i].CMSketch)
+		} else {
+			require.True(t, a.Columns[i].CMSketch.Equal(b.Columns[i].CMSketch))
+		}
+		// The nil case has been considered in (*TopN).Equal() so we don't need to consider it here.
+		require.Truef(t, a.Columns[i].TopN.Equal(b.Columns[i].TopN), "%v, %v", a.Columns[i].TopN, b.Columns[i].TopN)
+	}
+	require.Len(t, a.Indices, len(b.Indices))
+	for i := range a.Indices {
+		require.True(t, statistics.HistogramEqual(&a.Indices[i].Histogram, &b.Indices[i].Histogram, false))
+		if a.Indices[i].CMSketch == nil {
+			require.Nil(t, b.Indices[i].CMSketch)
+		} else {
+			require.True(t, a.Indices[i].CMSketch.Equal(b.Indices[i].CMSketch))
+		}
+		require.True(t, a.Indices[i].TopN.Equal(b.Indices[i].TopN))
+	}
+	require.True(t, IsSameExtendedStats(a.ExtendedStats, b.ExtendedStats))
+}
+
+// IsSameExtendedStats is to judge whether the extended states is the same.
+func IsSameExtendedStats(a, b *statistics.ExtendedStatsColl) bool {
+	aEmpty := (a == nil) || len(a.Stats) == 0
+	bEmpty := (b == nil) || len(b.Stats) == 0
+	if (aEmpty && !bEmpty) || (!aEmpty && bEmpty) {
+		return false
+	}
+	if aEmpty && bEmpty {
+		return true
+	}
+	if len(a.Stats) != len(b.Stats) {
+		return false
+	}
+	for aKey, aItem := range a.Stats {
+		bItem, ok := b.Stats[aKey]
+		if !ok {
+			return false
+		}
+		for i, id := range aItem.ColIDs {
+			if id != bItem.ColIDs[i] {
+				return false
+			}
+		}
+		if (aItem.Tp != bItem.Tp) || (aItem.ScalarVals != bItem.ScalarVals) || (aItem.StringVals != bItem.StringVals) {
+			return false
+		}
+	}
+	return true
+}