From d650863ddae365ba735ecb79fce69bf2372ac1db Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Fri, 8 Jan 2021 09:32:07 -0500 Subject: [PATCH] safely handle existing net namespace in default network manager When a client restarts, the network_hook's prerun will call `CreateNetwork`. Drivers that don't implement their own network manager will fall back to the default network manager, which doesn't handle the case where the network namespace is being recreated safely. This results in an error and the task being restarted for `exec` tasks with `network` blocks (this also impacts the community `containerd` and probably other community task drivers). If we get an error when attempting to create the namespace and that error is because the file already exists and is locked by its process, then we'll return a `nil` error with the `created` flag set to false, just as we do with the `docker` driver. --- CHANGELOG.md | 1 + client/allocrunner/network_manager_linux.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e26e66058da..fbea9d67a8f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ IMPROVEMENTS: * consul/connect: interpolate the connect, service meta, and service canary meta blocks with the task environment [[GH-9586](https://github.com/hashicorp/nomad/pull/9586)] BUG FIXES: + * client: Fixed a bug where non-`docker` tasks with network isolation were restarted on client restart. [[GH-9757](https://github.com/hashicorp/nomad/issues/9757)] * client: Fixed a bug where clients configured with `cpu_total_compute` did not update the `cpu.totalcompute` node attribute. [[GH-9532](https://github.com/hashicorp/nomad/issues/9532)] * core: Fixed a bug where an in place update dropped an allocations shared allocated resources [[GH-9736](https://github.com/hashicorp/nomad/issues/9736)] * consul: Fixed a bug where updating a task to include services would not work [[GH-9707](https://github.com/hashicorp/nomad/issues/9707)] diff --git a/client/allocrunner/network_manager_linux.go b/client/allocrunner/network_manager_linux.go index 63d63a34799f..b5bde8eff61e 100644 --- a/client/allocrunner/network_manager_linux.go +++ b/client/allocrunner/network_manager_linux.go @@ -2,7 +2,10 @@ package allocrunner import ( "fmt" + "os" + "path" "strings" + "syscall" hclog "github.com/hashicorp/go-hclog" clientconfig "github.com/hashicorp/nomad/client/config" @@ -92,6 +95,15 @@ type defaultNetworkManager struct{} func (*defaultNetworkManager) CreateNetwork(allocID string) (*drivers.NetworkIsolationSpec, bool, error) { netns, err := nsutil.NewNS(allocID) if err != nil { + // when a client restarts, the namespace will already exist and + // there will be a namespace file in use by the task process + if e, ok := err.(*os.PathError); ok && e.Err == syscall.EPERM { + nsPath := path.Join(nsutil.NetNSRunDir, allocID) + _, err := os.Stat(nsPath) + if err == nil { + return nil, false, nil + } + } return nil, false, err }