From f3d64ac13adeea9cf26a29e1179a93bbca5947bb Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 16 Mar 2022 15:48:57 -0500 Subject: [PATCH] wip better docker --- client/allocrunner/taskrunner/task_runner.go | 2 -- drivers/docker/config.go | 6 ------ drivers/docker/driver.go | 5 ++++- drivers/docker/fingerprint.go | 5 ++--- drivers/docker/reconcile_cpuset.go | 21 ++++++++++---------- drivers/exec/driver.go | 2 -- drivers/shared/executor/executor_linux.go | 1 - 7 files changed, 17 insertions(+), 25 deletions(-) diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 09e7ef5cad94..3cf751cf12b4 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -772,12 +772,10 @@ func (tr *TaskRunner) runDriver() error { taskConfig := tr.buildTaskConfig() if tr.cpusetCgroupPathGetter != nil { - fmt.Println("TaskRunner.runDriver, will wait for cpuset group") cpusetCgroupPath, err := tr.cpusetCgroupPathGetter(tr.killCtx) if err != nil { return err } - fmt.Println("TaskRunner.runDriver, SET cpuset group:", cpusetCgroupPath) taskConfig.Resources.LinuxResources.CpusetCgroupPath = cpusetCgroupPath } diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 9ac36a955a0a..6194d60f1e46 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -10,7 +10,6 @@ import ( docker "github.com/fsouza/go-dockerclient" "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/client/lib/cgutil" "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/pluginutils/loader" @@ -214,8 +213,6 @@ var ( } }`)), - "cgroup_parent": hclspec.NewAttr("cgroup_parent", "string", false), - // garbage collection options // default needed for both if the gc {...} block is not set and // if the default fields are missing @@ -631,7 +628,6 @@ type DriverConfig struct { pullActivityTimeoutDuration time.Duration `codec:"-"` ExtraLabels []string `codec:"extra_labels"` Logging LoggingConfig `codec:"logging"` - CgroupParent string `codec:"cgroup_parent"` AllowRuntimesList []string `codec:"allow_runtimes"` allowRuntimes map[string]struct{} `codec:"-"` @@ -689,8 +685,6 @@ func (d *Driver) SetConfig(c *base.Config) error { d.config = &config d.config.InfraImage = strings.TrimPrefix(d.config.InfraImage, "https://") - d.config.CgroupParent = cgutil.GetCgroupParent(d.config.CgroupParent) - if len(d.config.GC.ImageDelay) > 0 { dur, err := time.ParseDuration(d.config.GC.ImageDelay) if err != nil { diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 26a324f6451b..4696dd6bdbf5 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -851,8 +851,11 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T pidsLimit = driverConfig.PidsLimit } + // Extract the cgroup parent from the nomad cgroup (bypass the need for plugin config) + parent, _ := cgutil.SplitPath(task.Resources.LinuxResources.CpusetCgroupPath) + hostConfig := &docker.HostConfig{ - CgroupParent: d.config.CgroupParent, + CgroupParent: parent, Memory: memory, // hard limit MemoryReservation: memoryReservation, // soft limit diff --git a/drivers/docker/fingerprint.go b/drivers/docker/fingerprint.go index 3ba8f99fa91a..f37a169788e1 100644 --- a/drivers/docker/fingerprint.go +++ b/drivers/docker/fingerprint.go @@ -2,7 +2,6 @@ package docker import ( "context" - "fmt" "runtime" "sort" "strings" @@ -14,8 +13,8 @@ import ( ) func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { - fmt.Println("docker/Driver.Fingerprint") - // start docker reconcilers when we start fingerprinting + // Start docker reconcilers when we start fingerprinting, a workaround for + // task drivers not having a kind of post-setup hook. d.danglingReconciler.Start() d.cpusetFixer.Start() diff --git a/drivers/docker/reconcile_cpuset.go b/drivers/docker/reconcile_cpuset.go index fe614ffd0edb..85ab7a821939 100644 --- a/drivers/docker/reconcile_cpuset.go +++ b/drivers/docker/reconcile_cpuset.go @@ -32,9 +32,7 @@ type cpusetFixer struct { logger hclog.Logger interval time.Duration once sync.Once - parent string - - tasks func() map[coordinate]struct{} + tasks func() map[coordinate]struct{} } func newCpusetFixer(d *Driver) *cpusetFixer { @@ -42,7 +40,6 @@ func newCpusetFixer(d *Driver) *cpusetFixer { interval: cpusetReconcileInterval, ctx: d.ctx, logger: d.logger, - parent: d.config.CgroupParent, tasks: d.trackedTasks, } } @@ -83,8 +80,8 @@ func (cf *cpusetFixer) scan() { } func (cf *cpusetFixer) fix(c coordinate) { - source := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.NomadScope()) - destination := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.DockerScope()) + source := c.NomadCgroup() + destination := c.DockerCgroup() if err := cgutil.CopyCpuset(source, destination); err != nil { cf.logger.Trace("failed to copy cpuset", "err", err) } @@ -94,14 +91,17 @@ type coordinate struct { containerID string allocID string task string + path string } -func (c coordinate) NomadScope() string { - return cgutil.CgroupID(c.allocID, c.task) +func (c coordinate) NomadCgroup() string { + parent, _ := cgutil.SplitPath(c.path) + return filepath.Join(cgutil.V2CgroupRoot, parent, cgutil.CgroupID(c.allocID, c.task)) } -func (c coordinate) DockerScope() string { - return fmt.Sprintf("docker-%s.scope", c.containerID) +func (c coordinate) DockerCgroup() string { + parent, _ := cgutil.SplitPath(c.path) + return filepath.Join(cgutil.V2CgroupRoot, parent, fmt.Sprintf("docker-%s.scope", c.containerID)) } func (d *Driver) trackedTasks() map[coordinate]struct{} { @@ -114,6 +114,7 @@ func (d *Driver) trackedTasks() map[coordinate]struct{} { containerID: h.containerID, allocID: h.task.AllocID, task: h.task.Name, + path: h.task.Resources.LinuxResources.CpusetCgroupPath, }] = struct{}{} } return m diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index dc9481175aaf..422b8b42aab9 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -500,8 +500,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive Capabilities: caps, } - fmt.Println("SH driver.StartTask, exec:", execCmd.Cmd, execCmd.Resources.LinuxResources.CpusetCgroupPath) - ps, err := exec.Launch(execCmd) if err != nil { pluginClient.Kill() diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 79b5aab7fe4e..df276995852e 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -887,7 +887,6 @@ func lookPathIn(path string, root string, bin string) (string, error) { } func newSetCPUSetCgroupHook(cgroupPath string) lconfigs.Hook { - fmt.Println("SH newSetCPUSetCgroupHook, cgroupPath:", cgroupPath) return lconfigs.NewFunctionHook(func(state *specs.State) error { return cgroups.WriteCgroupProc(cgroupPath, state.Pid) })