wip better docker

client/allocrunner/taskrunner/task_runner.go

@@ -772,12 +772,10 @@ func (tr *TaskRunner) runDriver() error {
 
 	taskConfig := tr.buildTaskConfig()
 	if tr.cpusetCgroupPathGetter != nil {
-		fmt.Println("TaskRunner.runDriver, will wait for cpuset group")
 		cpusetCgroupPath, err := tr.cpusetCgroupPathGetter(tr.killCtx)
 		if err != nil {
 			return err
 		}
-		fmt.Println("TaskRunner.runDriver, SET cpuset group:", cpusetCgroupPath)
 		taskConfig.Resources.LinuxResources.CpusetCgroupPath = cpusetCgroupPath
 	}
 

drivers/docker/config.go

@@ -10,7 +10,6 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/nomad/client/lib/cgutil"
 	"github.com/hashicorp/nomad/drivers/shared/capabilities"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
@@ -214,8 +213,6 @@ var (
 			}
 		}`)),
 
-		"cgroup_parent": hclspec.NewAttr("cgroup_parent", "string", false),
-
 		// garbage collection options
 		// default needed for both if the gc {...} block is not set and
 		// if the default fields are missing
@@ -631,7 +628,6 @@ type DriverConfig struct {
 	pullActivityTimeoutDuration   time.Duration `codec:"-"`
 	ExtraLabels                   []string      `codec:"extra_labels"`
 	Logging                       LoggingConfig `codec:"logging"`
-	CgroupParent                  string        `codec:"cgroup_parent"`
 
 	AllowRuntimesList []string            `codec:"allow_runtimes"`
 	allowRuntimes     map[string]struct{} `codec:"-"`
@@ -689,8 +685,6 @@ func (d *Driver) SetConfig(c *base.Config) error {
 	d.config = &config
 	d.config.InfraImage = strings.TrimPrefix(d.config.InfraImage, "https://")
 
-	d.config.CgroupParent = cgutil.GetCgroupParent(d.config.CgroupParent)
-
 	if len(d.config.GC.ImageDelay) > 0 {
 		dur, err := time.ParseDuration(d.config.GC.ImageDelay)
 		if err != nil {

drivers/docker/driver.go

@@ -851,8 +851,11 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 		pidsLimit = driverConfig.PidsLimit
 	}
 
+	// Extract the cgroup parent from the nomad cgroup (bypass the need for plugin config)
+	parent, _ := cgutil.SplitPath(task.Resources.LinuxResources.CpusetCgroupPath)
+
 	hostConfig := &docker.HostConfig{
-		CgroupParent: d.config.CgroupParent,
+		CgroupParent: parent,
 
 		Memory:            memory,            // hard limit
 		MemoryReservation: memoryReservation, // soft limit

drivers/docker/fingerprint.go

@@ -2,7 +2,6 @@ package docker
 
 import (
 	"context"
-	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -14,8 +13,8 @@ import (
 )
 
 func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
-	fmt.Println("docker/Driver.Fingerprint")
-	// start docker reconcilers when we start fingerprinting
+	// Start docker reconcilers when we start fingerprinting, a workaround for
+	// task drivers not having a kind of post-setup hook.
 	d.danglingReconciler.Start()
 	d.cpusetFixer.Start()
 

drivers/docker/reconcile_cpuset.go

@@ -32,17 +32,14 @@ type cpusetFixer struct {
 	logger   hclog.Logger
 	interval time.Duration
 	once     sync.Once
-	parent   string
-
-	tasks func() map[coordinate]struct{}
+	tasks    func() map[coordinate]struct{}
 }
 
 func newCpusetFixer(d *Driver) *cpusetFixer {
 	return &cpusetFixer{
 		interval: cpusetReconcileInterval,
 		ctx:      d.ctx,
 		logger:   d.logger,
-		parent:   d.config.CgroupParent,
 		tasks:    d.trackedTasks,
 	}
 }
@@ -83,8 +80,8 @@ func (cf *cpusetFixer) scan() {
 }
 
 func (cf *cpusetFixer) fix(c coordinate) {
-	source := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.NomadScope())
-	destination := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.DockerScope())
+	source := c.NomadCgroup()
+	destination := c.DockerCgroup()
 	if err := cgutil.CopyCpuset(source, destination); err != nil {
 		cf.logger.Trace("failed to copy cpuset", "err", err)
 	}
@@ -94,14 +91,17 @@ type coordinate struct {
 	containerID string
 	allocID     string
 	task        string
+	path        string
 }
 
-func (c coordinate) NomadScope() string {
-	return cgutil.CgroupID(c.allocID, c.task)
+func (c coordinate) NomadCgroup() string {
+	parent, _ := cgutil.SplitPath(c.path)
+	return filepath.Join(cgutil.V2CgroupRoot, parent, cgutil.CgroupID(c.allocID, c.task))
 }
 
-func (c coordinate) DockerScope() string {
-	return fmt.Sprintf("docker-%s.scope", c.containerID)
+func (c coordinate) DockerCgroup() string {
+	parent, _ := cgutil.SplitPath(c.path)
+	return filepath.Join(cgutil.V2CgroupRoot, parent, fmt.Sprintf("docker-%s.scope", c.containerID))
 }
 
 func (d *Driver) trackedTasks() map[coordinate]struct{} {
@@ -114,6 +114,7 @@ func (d *Driver) trackedTasks() map[coordinate]struct{} {
 			containerID: h.containerID,
 			allocID:     h.task.AllocID,
 			task:        h.task.Name,
+			path:        h.task.Resources.LinuxResources.CpusetCgroupPath,
 		}] = struct{}{}
 	}
 	return m

drivers/exec/driver.go

@@ -500,8 +500,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		Capabilities:     caps,
 	}
 
-	fmt.Println("SH driver.StartTask, exec:", execCmd.Cmd, execCmd.Resources.LinuxResources.CpusetCgroupPath)
-
 	ps, err := exec.Launch(execCmd)
 	if err != nil {
 		pluginClient.Kill()

drivers/shared/executor/executor_linux.go

@@ -887,7 +887,6 @@ func lookPathIn(path string, root string, bin string) (string, error) {
 }
 
 func newSetCPUSetCgroupHook(cgroupPath string) lconfigs.Hook {
-	fmt.Println("SH newSetCPUSetCgroupHook, cgroupPath:", cgroupPath)
 	return lconfigs.NewFunctionHook(func(state *specs.State) error {
 		return cgroups.WriteCgroupProc(cgroupPath, state.Pid)
 	})