diff --git a/.changelog/12009.txt b/.changelog/12009.txt new file mode 100644 index 000000000000..e57cbca6a451 --- /dev/null +++ b/.changelog/12009.txt @@ -0,0 +1,3 @@ +```release-note:improvement +scheduler: recover scheduler goroutines on panic +``` diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index f263d864b246..2d83caad1954 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -125,7 +125,14 @@ func NewBatchScheduler(logger log.Logger, eventsCh chan<- interface{}, state Sta } // Process is used to handle a single evaluation -func (s *GenericScheduler) Process(eval *structs.Evaluation) error { +func (s *GenericScheduler) Process(eval *structs.Evaluation) (err error) { + + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("processing eval %q panicked scheduler: %v", eval.ID, r) + } + }() + // Store the evaluation s.eval = eval diff --git a/scheduler/scheduler_system.go b/scheduler/scheduler_system.go index dff79aa8bc2f..fefdc0101cf7 100644 --- a/scheduler/scheduler_system.go +++ b/scheduler/scheduler_system.go @@ -72,7 +72,13 @@ func NewSysBatchScheduler(logger log.Logger, eventsCh chan<- interface{}, state } // Process is used to handle a single evaluation. -func (s *SystemScheduler) Process(eval *structs.Evaluation) error { +func (s *SystemScheduler) Process(eval *structs.Evaluation) (err error) { + + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("processing eval %q panicked scheduler: %v", eval.ID, r) + } + }() // Store the evaluation s.eval = eval