Merge branch 'master' into export-config

.github/workflows/pd-tests.yaml

@@ -31,15 +31,23 @@ jobs:
           - worker_id: 2
             name: 'Unit Test(2)'
           - worker_id: 3
-            name: 'Tools Test'
+            name: 'Unit Test(3)'
           - worker_id: 4
-            name: 'Client Integration Test'
+            name: 'Tests(1)'
           - worker_id: 5
-            name: 'TSO Integration Test'
+            name: 'Tests(2)'
           - worker_id: 6
-            name: 'MicroService Integration Test'
+            name: 'Tools Test'
+          - worker_id: 7
+            name: 'Client Integration Test'
+          - worker_id: 8
+            name: 'TSO Integration Test'
+          - worker_id: 9
+            name: 'MicroService Integration(!TSO)'
+          - worker_id: 10
+            name: 'MicroService Integration(TSO)'
     outputs:
-      job-total: 6
+      job-total: 10
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -52,6 +60,9 @@ jobs:
         run: |
           make ci-test-job JOB_INDEX=$WORKER_ID
           mv covprofile covprofile_$WORKER_ID
+          if [ -f junitfile ]; then
+            cat junitfile
+          fi
       - name: Upload coverage result ${{ matrix.worker_id }}
         uses: actions/upload-artifact@v4
         with:
@@ -77,7 +88,7 @@ jobs:
           # only keep the first line(`mode: aomic`) of the coverage profile
           sed -i '2,${/mode: atomic/d;}' covprofile
       - name: Send coverage
-        uses: codecov/codecov-action@v4.2.0
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV }}
           file: ./covprofile

.gitignore

@@ -19,6 +19,7 @@ docs/swagger/*
 *.before
 covprofile
 coverage.tmp
+junitfile
 package.list
 report.xml
 coverage.xml

Makefile

@@ -228,7 +228,7 @@ failpoint-disable: install-tools
 ut: pd-ut
 	@$(FAILPOINT_ENABLE)
 	# only run unit tests
-	./bin/pd-ut run --ignore tests --race
+	./bin/pd-ut run --ignore tests --race --junitfile ./junitfile
 	@$(CLEAN_UT_BINARY)
 	@$(FAILPOINT_DISABLE)
 

client/tlsutil/tlsconfig.go

@@ -47,21 +47,12 @@ import (
 type TLSInfo struct {
 	CertFile           string
 	KeyFile            string
-	CAFile             string // TODO: deprecate this in v4
 	TrustedCAFile      string
-	ClientCertAuth     bool
-	CRLFile            string
 	InsecureSkipVerify bool
 
-	SkipClientSANVerify bool
-
 	// ServerName ensures the cert matches the given host in case of discovery / virtual hosting
 	ServerName string
 
-	// HandshakeFailure is optionally called when a connection fails to handshake. The
-	// connection will be closed immediately afterwards.
-	HandshakeFailure func(*tls.Conn, error)
-
 	// CipherSuites is a list of supported cipher suites.
 	// If empty, Go auto-populates it by default.
 	// Note that cipher suites are prioritized in the given order.
@@ -157,9 +148,6 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) {
 // cafiles returns a list of CA file paths.
 func (info TLSInfo) cafiles() []string {
 	cs := make([]string, 0)
-	if info.CAFile != "" {
-		cs = append(cs, info.CAFile)
-	}
 	if info.TrustedCAFile != "" {
 		cs = append(cs, info.TrustedCAFile)
 	}

pkg/core/region_test.go

@@ -731,7 +731,7 @@ func BenchmarkRandomSetRegion(b *testing.B) {
 
 func TestGetRegionSizeByRange(t *testing.T) {
 	regions := NewRegionsInfo()
-	nums := 1000010
+	nums := 100001
 	for i := 0; i < nums; i++ {
 		peer := &metapb.Peer{StoreId: 1, Id: uint64(i + 1)}
 		endKey := []byte(fmt.Sprintf("%20d", i+1))

pkg/schedule/schedulers/scheduler_controller.go

@@ -478,8 +478,8 @@ retry:
 
 		// If we have schedule, reset interval to the minimal interval.
 		s.nextInterval = s.Scheduler.GetMinInterval()
-		for _, op := range ops {
-			region := s.cluster.GetRegion(op.RegionID())
+		for i := 0; i < len(ops); i++ {
+			region := s.cluster.GetRegion(ops[i].RegionID())
 			if region == nil {
 				continue retry
 			}
@@ -492,9 +492,13 @@ retry:
 			// Refer: https://docs.pingcap.com/tidb-in-kubernetes/stable/restart-a-tidb-cluster#perform-a-graceful-restart-to-a-single-tikv-pod
 			if labelMgr.ScheduleDisabled(region) && !isEvictLeaderScheduler {
 				denySchedulersByLabelerCounter.Inc()
-				continue retry
+				ops = append(ops[:i], ops[i+1:]...)
+				i--
 			}
 		}
+		if len(ops) == 0 {
+			continue
+		}
 		return ops
 	}
 	s.nextInterval = s.Scheduler.GetNextInterval(s.nextInterval)

scripts/ci-subtask.sh

@@ -3,34 +3,51 @@
 # ./ci-subtask.sh <TOTAL_TASK_N> <TASK_INDEX>
 
 ROOT_PATH_COV=$(pwd)/covprofile
+ROOT_PATH_JUNITFILE=$(pwd)/junitfile
 # Currently, we only have 3 integration tests, so we can hardcode the task index.
 integrations_dir=$(pwd)/tests/integrations
 
 case $1 in
     1)
-        # unit tests ignore `tests`
-        ./bin/pd-ut run --race --ignore tests --coverprofile $ROOT_PATH_COV || exit 1
+        # unit tests ignore `tests`, `server`, `schedule`,`utils` and `encryption`
+        ./bin/pd-ut run --ignore tests,server,pkg/schedule,pkg/utils,pkg/encryption --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
         ;;
     2)
-        # unit tests only in `tests`
-        ./bin/pd-ut run tests --race --coverprofile $ROOT_PATH_COV || exit 1
+        # unit tests only in `schedule` and `encryption` without `tests`
+        ./bin/pd-ut run schedule,encryption --ignore tests --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
         ;;
     3)
+        # unit tests only in `server` and `utils` without `tests`
+        ./bin/pd-ut run server,utils --ignore tests --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
+        ;;
+    4)
+        # tests only in `tests` without `server/api, server/cluster and `server/config`
+        ./bin/pd-ut run tests --ignore tests/server/api,tests/server/cluster,tests/server/config --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
+        ;;
+    5)
+        # tests only in `server/api, server/cluster and `server/config` in `tests`
+        ./bin/pd-ut run tests/server/api,tests/server/cluster,tests/server/config --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
+        ;;
+    6)
         # tools tests
         cd ./tools && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
         ;;
-    4)
+    7)
         # integration test client
-        ./bin/pd-ut it run client --race --coverprofile $ROOT_PATH_COV || exit 1
+        ./bin/pd-ut it run client --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
         # client tests
         cd ./client && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
         ;;
-    5)
+    8)
         # integration test tso
-        ./bin/pd-ut it run tso --race --coverprofile $ROOT_PATH_COV || exit 1
+        ./bin/pd-ut it run tso --race --ignore mcs/tso --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
         ;;
-    6)
-        # integration test mcs
-        ./bin/pd-ut it run mcs --race --coverprofile $ROOT_PATH_COV || exit 1
+    9)
+        # integration test mcs without mcs/tso
+        ./bin/pd-ut it run mcs --race --ignore mcs/tso --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
+        ;;
+    10)
+        # integration test mcs/tso
+        ./bin/pd-ut it run mcs/tso --race --coverprofile $ROOT_PATH_COV --junitfile $ROOT_PATH_JUNITFILE || exit 1
         ;;
 esac

server/api/region_test.go

@@ -333,7 +333,7 @@ func TestRegionsWithKillRequest(t *testing.T) {
 	url := fmt.Sprintf("%s%s/api/v1/regions", addr, apiPrefix)
 	mustBootstrapCluster(re, svr)
 
-	regionCount := 100000
+	regionCount := 10000
 	tu.GenerateTestDataConcurrently(regionCount, func(i int) {
 		r := core.NewTestRegionInfo(uint64(i+2), 1,
 			[]byte(fmt.Sprintf("%09d", i)),

tests/integrations/client/client_test.go

@@ -217,7 +217,7 @@ func TestLeaderTransferAndMoveCluster(t *testing.T) {
 	}()
 
 	// Transfer leader.
-	for i := 0; i < 5; i++ {
+	for i := 0; i < 3; i++ {
 		oldLeaderName := cluster.WaitLeader()
 		err := cluster.GetServer(oldLeaderName).ResignLeader()
 		re.NoError(err)

tests/integrations/mcs/scheduling/api_test.go

@@ -514,7 +514,7 @@ func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestClust
 }
 
 func (suite *apiTestSuite) TestFollowerForward() {
-	suite.env.RunTestInTwoModes(suite.checkFollowerForward)
+	suite.env.RunTestBasedOnMode(suite.checkFollowerForward)
 }
 
 func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) {

tests/integrations/mcs/tso/keyspace_group_manager_test.go

@@ -67,7 +67,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) allocID() uint32 {
 	return uint32(id)
 }
 
-func TestTSOKeyspaceGroupManager(t *testing.T) {
+func TestTSOKeyspaceGroupManagerSuite(t *testing.T) {
 	suite.Run(t, &tsoKeyspaceGroupManagerTestSuite{})
 }