wip docker cgroup_parent

client/lib/cgutil/cgutil_linux.go

@@ -51,9 +51,8 @@ func GetCPUsFromCgroup(group string) ([]uint16, error) {
 //
 // Handles the cgroup root if present.
 func SplitPath(p string) (string, string) {
-	p = strings.TrimPrefix(p, v2CgroupRoot)
-	p = strings.TrimPrefix(p, "/")
-	p = strings.TrimSuffix(p, "/")
+	p = strings.TrimPrefix(p, V2CgroupRoot)
+	p = strings.Trim(p, "/")
 	parts := strings.Split(p, string(os.PathSeparator))
 	return parts[0], "/" + filepath.Join(parts[1:]...)
 }

client/lib/cgutil/cpuset_manager_v2.go

@@ -26,8 +26,8 @@ const (
 	// configured systemd cgroup.
 	V2defaultCgroupParent = "nomad.slice"
 
-	// v2CgroupRoot is hard-coded in the cgroups.v2 specification.
-	v2CgroupRoot = "/sys/fs/cgroup"
+	// V2CgroupRoot is hard-coded in the cgroups.v2 specification.
+	V2CgroupRoot = "/sys/fs/cgroup"
 
 	// v2isRootless is (for now) always false; Nomad clients require root.
 	v2isRootless = false
@@ -65,7 +65,7 @@ func NewCpusetManagerV2(parent string, logger hclog.Logger) CpusetManager {
 	return &cpusetManagerV2{
 		ctx:       context.TODO(),
 		parent:    cgroupParent,
-		parentAbs: filepath.Join(v2CgroupRoot, cgroupParent),
+		parentAbs: filepath.Join(V2CgroupRoot, cgroupParent),
 		logger:    logger,
 		sharing:   make(map[identifier]nothing),
 		isolating: make(map[identifier]cpuset.CPUSet),
@@ -304,7 +304,7 @@ func (c *cpusetManagerV2) ensureParent() error {
 }
 
 func v2Root(group string) string {
-	return filepath.Join(v2CgroupRoot, group)
+	return filepath.Join(V2CgroupRoot, group)
 }
 
 func v2GetCPUsFromCgroup(group string) ([]uint16, error) {

command/agent/config.go

@@ -1763,6 +1763,11 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.BindWildcardDefaultHostNetwork {
 		result.BindWildcardDefaultHostNetwork = true
 	}
+
+	if b.CgroupParent != "" {
+		result.CgroupParent = b.CgroupParent
+	}
+
 	return &result
 }
 

drivers/docker/config.go

@@ -10,6 +10,7 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/client/lib/cgutil"
 	"github.com/hashicorp/nomad/drivers/shared/capabilities"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
@@ -213,6 +214,8 @@ var (
 			}
 		}`)),
 
+		"cgroup_parent": hclspec.NewAttr("cgroup_parent", "string", false),
+
 		// garbage collection options
 		// default needed for both if the gc {...} block is not set and
 		// if the default fields are missing
@@ -628,6 +631,7 @@ type DriverConfig struct {
 	pullActivityTimeoutDuration   time.Duration `codec:"-"`
 	ExtraLabels                   []string      `codec:"extra_labels"`
 	Logging                       LoggingConfig `codec:"logging"`
+	CgroupParent                  string        `codec:"cgroup_parent"`
 
 	AllowRuntimesList []string            `codec:"allow_runtimes"`
 	allowRuntimes     map[string]struct{} `codec:"-"`
@@ -685,6 +689,8 @@ func (d *Driver) SetConfig(c *base.Config) error {
 	d.config = &config
 	d.config.InfraImage = strings.TrimPrefix(d.config.InfraImage, "https://")
 
+	d.config.CgroupParent = cgutil.GetCgroupParent(d.config.CgroupParent)
+
 	if len(d.config.GC.ImageDelay) > 0 {
 		dur, err := time.ParseDuration(d.config.GC.ImageDelay)
 		if err != nil {

drivers/docker/driver.go

@@ -852,13 +852,13 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 	}
 
 	hostConfig := &docker.HostConfig{
+		CgroupParent: d.config.CgroupParent,
+
 		Memory:            memory,            // hard limit
 		MemoryReservation: memoryReservation, // soft limit
 
 		CPUShares: task.Resources.LinuxResources.CPUShares,
 
-		CgroupParent: "nomad.slice", // configurable
-
 		// Binds are used to mount a host volume into the container. We mount a
 		// local directory for storage and a shared alloc directory that can be
 		// used to share data between different tasks in the same task group.

drivers/docker/reconcile_cpuset.go

@@ -32,22 +32,26 @@ type cpusetFixer struct {
 	logger   hclog.Logger
 	interval time.Duration
 	once     sync.Once
+	parent   string
 
 	tasks func() map[coordinate]struct{}
 }
 
 func newCpusetFixer(d *Driver) *cpusetFixer {
-	fmt.Println("newCpusetFixer")
 	return &cpusetFixer{
 		interval: cpusetReconcileInterval,
 		ctx:      d.ctx,
 		logger:   d.logger,
+		parent:   d.config.CgroupParent,
 		tasks:    d.trackedTasks,
 	}
 }
 
+// Start will start the background cpuset reconciliation until the cf context is
+// cancelled for shutdown.
+//
+// Only runs if cgroups.v2 is in use.
 func (cf *cpusetFixer) Start() {
-	fmt.Println("cpusetFixer.Start")
 	cf.once.Do(func() {
 		if cgutil.UseV2 {
 			go cf.loop()
@@ -79,8 +83,8 @@ func (cf *cpusetFixer) scan() {
 }
 
 func (cf *cpusetFixer) fix(c coordinate) {
-	source := filepath.Join("/sys/fs/cgroup/nomad.slice", c.NomadScope())
-	destination := filepath.Join("/sys/fs/cgroup/nomad.slice", c.DockerScope())
+	source := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.NomadScope())
+	destination := filepath.Join(cgutil.V2CgroupRoot, cf.parent, c.DockerScope())
 	if err := cgutil.CopyCpuset(source, destination); err != nil {
 		cf.logger.Trace("failed to copy cpuset", "err", err)
 	}