Fix JobDispatcher crash during force cancellation. (#3118)

actions · Jan 30, 2024 · 5268d74 · 5268d74
1 parent 7414e08
commit 5268d74
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs
@@ -629,6 +629,20 @@ await processChannel.SendAsync(
                                     Trace.Info("worker process has been killed.");
                                 }
                             }
+                            catch (Exception ex)
+                            {
+                                // message send failed, this might indicate worker process is already exited or stuck.
+                                Trace.Info($"Job cancel message sending for job {message.JobId} failed, kill running worker. {ex}");
+                                workerProcessCancelTokenSource.Cancel();
+                                try
+                                {
+                                    await workerProcessTask;
+                                }
+                                catch (OperationCanceledException)
+                                {
+                                    Trace.Info("worker process has been killed.");
+                                }
+                            }
 
                             // wait worker to exit 
                             // if worker doesn't exit within timeout, then kill worker.