resourcemanager: fix TaskController.Stop() can't make producer exit i…

…n spmcpool (#41016)

close #41015

resourcemanager/pooltask/BUILD.bazel

@@ -10,7 +10,10 @@ go_library(
     ],
     importpath = "github.com/pingcap/tidb/resourcemanager/pooltask",
     visibility = ["//visibility:public"],
-    deps = ["@org_uber_go_atomic//:atomic"],
+    deps = [
+        "//util/channel",
+        "@org_uber_go_atomic//:atomic",
+    ],
 )
 
 go_test(

resourcemanager/pooltask/task.go

@@ -17,6 +17,8 @@ package pooltask
 import (
 	"sync"
 	"sync/atomic"
+
+	"github.com/pingcap/tidb/util/channel"
 )
 
 // Context is a interface that can be used to create a context.
@@ -127,35 +129,44 @@ type GPool[T any, U any, C any, CT any, TF Context[CT]] interface {
 
 // TaskController is a controller that can control or watch the pool.
 type TaskController[T any, U any, C any, CT any, TF Context[CT]] struct {
-	pool     GPool[T, U, C, CT, TF]
-	close    chan struct{}
-	wg       *sync.WaitGroup
-	taskID   uint64
-	resultCh chan U
+	pool          GPool[T, U, C, CT, TF]
+	productExitCh chan struct{}
+	wg            *sync.WaitGroup
+	taskID        uint64
+	resultCh      chan U
+	inputCh       chan Task[T]
 }
 
 // NewTaskController create a controller to deal with pooltask's status.
-func NewTaskController[T any, U any, C any, CT any, TF Context[CT]](p GPool[T, U, C, CT, TF], taskID uint64, closeCh chan struct{}, wg *sync.WaitGroup, resultCh chan U) TaskController[T, U, C, CT, TF] {
+func NewTaskController[T any, U any, C any, CT any, TF Context[CT]](p GPool[T, U, C, CT, TF], taskID uint64, productExitCh chan struct{}, wg *sync.WaitGroup, inputCh chan Task[T], resultCh chan U) TaskController[T, U, C, CT, TF] {
 	return TaskController[T, U, C, CT, TF]{
-		pool:     p,
-		taskID:   taskID,
-		close:    closeCh,
-		wg:       wg,
-		resultCh: resultCh,
+		pool:          p,
+		taskID:        taskID,
+		productExitCh: productExitCh,
+		wg:            wg,
+		resultCh:      resultCh,
+		inputCh:       inputCh,
 	}
 }
 
 // Wait is to wait the pool task to stop.
 func (t *TaskController[T, U, C, CT, TF]) Wait() {
-	<-t.close
 	t.wg.Wait()
 	close(t.resultCh)
 	t.pool.DeleteTask(t.taskID)
 }
 
 // Stop is to send stop command to the task. But you still need to wait the task to stop.
 func (t *TaskController[T, U, C, CT, TF]) Stop() {
+	close(t.productExitCh)
+	// Clear all the task in the task queue and mark all task complete.
+	// so that ```t.Wait``` is able to close resultCh
+	for range t.inputCh {
+		t.wg.Done()
+	}
 	t.pool.StopTask(t.TaskID())
+	// Clear the resultCh to avoid blocking the consumer put result into the channel and cannot exit.
+	channel.Clear(t.resultCh)
 }
 
 // TaskID is to get the task id.

util/gpool/spmc/option.go

@@ -18,6 +18,8 @@ import (
 	"time"
 )
 
+const defaultTaskChanLen = 1
+
 // Option represents the optional function.
 type Option func(opts *Options)
 
@@ -103,8 +105,8 @@ func loadTaskOptions(options ...TaskOption) *TaskOptions {
 	if opts.ResultChanLen == 0 {
 		opts.ResultChanLen = uint64(opts.Concurrency)
 	}
-	if opts.ResultChanLen == 0 {
-		opts.ResultChanLen = uint64(opts.Concurrency)
+	if opts.TaskChanLen == 0 {
+		opts.TaskChanLen = defaultTaskChanLen
 	}
 	return opts
 }

util/gpool/spmc/spmcpool.go

@@ -219,7 +219,6 @@ func (p *Pool[T, U, C, CT, TF]) release() {
 	// There might be some callers waiting in retrieveWorker(), so we need to wake them up to prevent
 	// those callers blocking infinitely.
 	p.cond.Broadcast()
-	close(p.taskCh)
 }
 
 func isClose(exitCh chan struct{}) bool {
@@ -260,9 +259,9 @@ func (p *Pool[T, U, C, CT, TF]) AddProduceBySlice(producer func() ([]T, error),
 	taskID := p.NewTaskID()
 	var wg sync.WaitGroup
 	result := make(chan U, opt.ResultChanLen)
-	closeCh := make(chan struct{})
+	productExitCh := make(chan struct{})
 	inputCh := make(chan pooltask.Task[T], opt.TaskChanLen)
-	tc := pooltask.NewTaskController[T, U, C, CT, TF](p, taskID, closeCh, &wg, result)
+	tc := pooltask.NewTaskController[T, U, C, CT, TF](p, taskID, productExitCh, &wg, inputCh, result)
 	p.taskManager.RegisterTask(taskID, int32(opt.Concurrency))
 	for i := 0; i < opt.Concurrency; i++ {
 		err := p.run()
@@ -274,15 +273,19 @@ func (p *Pool[T, U, C, CT, TF]) AddProduceBySlice(producer func() ([]T, error),
 		p.taskManager.AddSubTask(taskID, &taskBox)
 		p.taskCh <- &taskBox
 	}
+	wg.Add(1)
 	go func() {
 		defer func() {
 			if r := recover(); r != nil {
 				logutil.BgLogger().Error("producer panic", zap.Any("recover", r), zap.Stack("stack"))
 			}
-			close(closeCh)
 			close(inputCh)
+			wg.Done()
 		}()
 		for {
+			if isClose(productExitCh) {
+				return
+			}
 			tasks, err := producer()
 			if err != nil {
 				if errors.Is(err, gpool.ErrProducerClosed) {
@@ -310,10 +313,10 @@ func (p *Pool[T, U, C, CT, TF]) AddProducer(producer func() (T, error), constArg
 	taskID := p.NewTaskID()
 	var wg sync.WaitGroup
 	result := make(chan U, opt.ResultChanLen)
-	closeCh := make(chan struct{})
+	productExitCh := make(chan struct{})
 	inputCh := make(chan pooltask.Task[T], opt.TaskChanLen)
 	p.taskManager.RegisterTask(taskID, int32(opt.Concurrency))
-	tc := pooltask.NewTaskController[T, U, C, CT, TF](p, taskID, closeCh, &wg, result)
+	tc := pooltask.NewTaskController[T, U, C, CT, TF](p, taskID, productExitCh, &wg, inputCh, result)
 	for i := 0; i < opt.Concurrency; i++ {
 		err := p.run()
 		if err == gpool.ErrPoolClosed {
@@ -324,15 +327,19 @@ func (p *Pool[T, U, C, CT, TF]) AddProducer(producer func() (T, error), constArg
 		p.taskManager.AddSubTask(taskID, &taskBox)
 		p.taskCh <- &taskBox
 	}
+	wg.Add(1)
 	go func() {
 		defer func() {
 			if r := recover(); r != nil {
 				logutil.BgLogger().Error("producer panic", zap.Any("recover", r), zap.Stack("stack"))
 			}
-			close(closeCh)
 			close(inputCh)
+			wg.Done()
 		}()
 		for {
+			if isClose(productExitCh) {
+				return
+			}
 			task, err := producer()
 			if err != nil {
 				if errors.Is(err, gpool.ErrProducerClosed) {

util/gpool/spmc/spmcpool_test.go

@@ -53,7 +53,7 @@ func TestPool(t *testing.T) {
 		}
 	}
 	// add new task
-	resultCh, control := pool.AddProducer(pfunc, myArgs, pooltask.NilContext{}, WithConcurrency(4))
+	resultCh, control := pool.AddProducer(pfunc, myArgs, pooltask.NilContext{}, WithConcurrency(5))
 
 	var count atomic.Uint32
 	var wg sync.WaitGroup
@@ -112,12 +112,55 @@ func TestStopPool(t *testing.T) {
 			require.Greater(t, result, 10)
 		}
 	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		control.Stop()
+	}()
+	// Waiting task finishing
+	control.Wait()
+	wg.Wait()
+	// close pool
+	pool.ReleaseAndWait()
+}
+
+func TestStopPoolWithSlice(t *testing.T) {
+	type ConstArgs struct {
+		a int
+	}
+	myArgs := ConstArgs{a: 10}
+	// init the pool
+	// input type， output type, constArgs type
+	pool, err := NewSPMCPool[int, int, ConstArgs, any, pooltask.NilContext]("TestStopPoolWithSlice", 3, rmutil.UNKNOWN)
+	require.NoError(t, err)
+	pool.SetConsumerFunc(func(task int, constArgs ConstArgs, ctx any) int {
+		return task + constArgs.a
+	})
+
+	exit := make(chan struct{})
+
+	pfunc := func() ([]int, error) {
+		select {
+		case <-exit:
+			return nil, gpool.ErrProducerClosed
+		default:
+			return []int{1, 2, 3}, nil
+		}
+	}
+	// add new task
+	resultCh, control := pool.AddProduceBySlice(pfunc, myArgs, pooltask.NilContext{}, WithConcurrency(4))
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for result := range resultCh {
+			require.Greater(t, result, 10)
+			control.Stop()
+		}
+	}()
 	// Waiting task finishing
-	control.Stop()
-	close(exit)
 	control.Wait()
-	// it should pass. Stop can be used after the pool is closed. we should prevent it from panic.
-	control.Stop()
 	wg.Wait()
 	// close pool
 	pool.ReleaseAndWait()
@@ -191,9 +234,12 @@ func testTunePool(t *testing.T, name string) {
 	for n := pool.Cap(); n > 1; n-- {
 		downclockPool(t, pool, tid)
 	}
-
-	// exit test
-	close(exit)
+	wg.Add(1)
+	go func() {
+		// exit test
+		control.Stop()
+		wg.Done()
+	}()
 	control.Wait()
 	wg.Wait()
 	// close pool