From 054e3830dba206872656850dd897a0b9e914e000 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Wed, 22 Dec 2021 16:49:56 -0500 Subject: [PATCH] task runner: fix goroutine leak in prestart hook The task runner prestart hooks take a `joincontext` so they have the option to exit early if either of two contexts are canceled: from killing the task or client shutdown. Some tasks exit without being shutdown from the server, so neither of the joined contexts ever gets canceled and we leak the `joincontext` (48 bytes) and its internal goroutine. This primarily impacts batch jobs and any task that fails or completes early such as non-sidecar prestart lifecycle tasks. Cancel the `joincontext` after the prestart call exits to fix the leak. --- .changelog/11741.txt | 3 +++ client/allocrunner/taskrunner/task_runner_hooks.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changelog/11741.txt diff --git a/.changelog/11741.txt b/.changelog/11741.txt new file mode 100644 index 000000000000..1302fc2e3e39 --- /dev/null +++ b/.changelog/11741.txt @@ -0,0 +1,3 @@ +```release-note:bug +client: Fixed a memory and goroutine leak for batch tasks and any task that exits without being shut down from the server +``` diff --git a/client/allocrunner/taskrunner/task_runner_hooks.go b/client/allocrunner/taskrunner/task_runner_hooks.go index c9ff34441ed5..72a509e056c7 100644 --- a/client/allocrunner/taskrunner/task_runner_hooks.go +++ b/client/allocrunner/taskrunner/task_runner_hooks.go @@ -237,7 +237,9 @@ func (tr *TaskRunner) prestart() error { // Run the prestart hook // use a joint context to allow any blocking pre-start hooks // to be canceled by either killCtx or shutdownCtx - joinedCtx, _ := joincontext.Join(tr.killCtx, tr.shutdownCtx) + joinedCtx, joinedCancel := joincontext.Join(tr.killCtx, tr.shutdownCtx) + defer joinedCancel() + var resp interfaces.TaskPrestartResponse if err := pre.Prestart(joinedCtx, &req, &resp); err != nil { tr.emitHookError(err, name)