-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
raw_exec: make raw exec driver work with cgroups v2 #12419
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
```release-note:improvement | ||
Add support for cgroups v2 in raw_exec driver | ||
``` |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,210 @@ | ||
//go:build linux | ||
|
||
package cgutil | ||
|
||
import ( | ||
"errors" | ||
"fmt" | ||
"os" | ||
"path/filepath" | ||
"time" | ||
|
||
"github.com/hashicorp/go-hclog" | ||
"github.com/opencontainers/runc/libcontainer/cgroups" | ||
"github.com/opencontainers/runc/libcontainer/cgroups/fs" | ||
"github.com/opencontainers/runc/libcontainer/cgroups/fs2" | ||
"github.com/opencontainers/runc/libcontainer/configs" | ||
) | ||
|
||
// freezer is the name of the cgroup subsystem used for stopping / starting | ||
// a group of processes | ||
const freezer = "freezer" | ||
|
||
// thawed and frozen are the two states we put a cgroup in when trying to remove it | ||
var ( | ||
thawed = &configs.Resources{Freezer: configs.Thawed} | ||
frozen = &configs.Resources{Freezer: configs.Frozen} | ||
) | ||
|
||
// GroupKiller is used for SIGKILL-ing the process tree[s] of a cgroup by leveraging | ||
// the freezer cgroup subsystem. | ||
type GroupKiller interface { | ||
KillGroup(cgroup *configs.Cgroup) error | ||
} | ||
|
||
// NewGroupKiller creates a GroupKiller with executor PID pid. | ||
func NewGroupKiller(logger hclog.Logger, pid int) GroupKiller { | ||
return &killer{ | ||
logger: logger.Named("group_killer"), | ||
pid: pid, | ||
} | ||
} | ||
|
||
type killer struct { | ||
logger hclog.Logger | ||
pid int | ||
} | ||
|
||
// KillGroup will SIGKILL the process tree present in cgroup, using the freezer | ||
// subsystem to prevent further forking, etc. | ||
func (d *killer) KillGroup(cgroup *configs.Cgroup) error { | ||
if UseV2 { | ||
return d.v2(cgroup) | ||
} | ||
return d.v1(cgroup) | ||
} | ||
|
||
func (d *killer) v1(cgroup *configs.Cgroup) error { | ||
if cgroup == nil { | ||
return errors.New("missing cgroup") | ||
} | ||
|
||
// the actual path to our tasks freezer cgroup | ||
path := cgroup.Paths[freezer] | ||
|
||
d.logger.Trace("killing processes", "cgroup_path", path, "cgroup_version", "v1", "executor_pid", d.pid) | ||
|
||
// move executor PID into the init freezer cgroup so we can kill the task | ||
// pids without killing the executor (which is the process running this code, | ||
// doing the killing) | ||
initPath, err := cgroups.GetInitCgroupPath(freezer) | ||
if err != nil { | ||
return fmt.Errorf("failed to find init cgroup: %w", err) | ||
} | ||
m := map[string]string{freezer: initPath} | ||
if err = cgroups.EnterPid(m, d.pid); err != nil { | ||
return fmt.Errorf("failed to add executor pid to init cgroup: %w", err) | ||
} | ||
|
||
// ability to freeze the cgroup | ||
freeze := func() { | ||
_ = new(fs.FreezerGroup).Set(path, frozen) | ||
} | ||
|
||
// ability to thaw the cgroup | ||
thaw := func() { | ||
_ = new(fs.FreezerGroup).Set(path, thawed) | ||
} | ||
|
||
// do the common kill logic | ||
if err = d.kill(path, freeze, thaw); err != nil { | ||
return err | ||
} | ||
|
||
// remove the cgroup from disk | ||
return cgroups.RemovePath(path) | ||
} | ||
|
||
func (d *killer) v2(cgroup *configs.Cgroup) error { | ||
if cgroup == nil { | ||
return errors.New("missing cgroup") | ||
} | ||
|
||
path := filepath.Join(CgroupRoot, cgroup.Path) | ||
|
||
existingPIDs, err := cgroups.GetPids(path) | ||
if err != nil { | ||
return fmt.Errorf("failed to determine pids in cgroup: %w", err) | ||
} | ||
|
||
d.logger.Trace("killing processes", "cgroup_path", path, "cgroup_version", "v2", "executor_pid", d.pid, "existing_pids", existingPIDs) | ||
|
||
mgr, err := fs2.NewManager(cgroup, "", rootless) | ||
if err != nil { | ||
return fmt.Errorf("failed to create v2 cgroup manager: %w", err) | ||
} | ||
|
||
// move executor PID into the root init.scope so we can kill the task pids | ||
// without killing the executor (which is the process running this code, doing | ||
// the killing) | ||
init, err := fs2.NewManager(nil, filepath.Join(CgroupRoot, "init.scope"), rootless) | ||
if err != nil { | ||
return fmt.Errorf("failed to create v2 init cgroup manager: %w", err) | ||
} | ||
if err = init.Apply(d.pid); err != nil { | ||
return fmt.Errorf("failed to move executor pid into init.scope cgroup: %w", err) | ||
} | ||
|
||
d.logger.Trace("move of executor pid into init.scope complete", "pid", d.pid) | ||
|
||
// ability to freeze the cgroup | ||
freeze := func() { | ||
_ = mgr.Freeze(configs.Frozen) | ||
} | ||
|
||
// ability to thaw the cgroup | ||
thaw := func() { | ||
_ = mgr.Freeze(configs.Thawed) | ||
} | ||
|
||
// do the common kill logic | ||
|
||
if err = d.kill(path, freeze, thaw); err != nil { | ||
return err | ||
} | ||
|
||
// remove the cgroup from disk | ||
return mgr.Destroy() | ||
} | ||
|
||
// kill is used to SIGKILL all processes in cgroup | ||
// | ||
// The order of operations is | ||
// 0. before calling this method, the executor pid has been moved outside of cgroup | ||
// 1. freeze cgroup (so processes cannot fork further) | ||
// 2. scan the cgroup to collect all pids | ||
// 3. issue SIGKILL to each pid found | ||
// 4. thaw the cgroup so processes can go die | ||
// 5. wait on each processes until it is confirmed dead | ||
func (d *killer) kill(cgroup string, freeze func(), thaw func()) error { | ||
// freeze the cgroup stopping further forking | ||
freeze() | ||
|
||
d.logger.Trace("search for pids in", "cgroup", cgroup) | ||
|
||
// find all the pids we intend to kill | ||
pids, err := cgroups.GetPids(cgroup) | ||
if err != nil { | ||
// if we fail to get pids, re-thaw before bailing so there is at least | ||
// a chance the processes can go die out of band | ||
thaw() | ||
return fmt.Errorf("failed to find pids: %w", err) | ||
} | ||
|
||
d.logger.Trace("send sigkill to frozen processes", "cgroup", cgroup, "pids", pids) | ||
|
||
var processes []*os.Process | ||
|
||
// kill the processes in cgroup | ||
for _, pid := range pids { | ||
Comment on lines
+178
to
+179
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we not expect that all the PIDs are going to be in the same process group? That is, could we kill via There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is honestly just copying the original implementation, but I believe this defends against sub-sub-sub processes that get daemonized. So like Executor -> A -> B -> C If B dies C gets owed by PID 1, not A and is no longer part of the process group ... right? 🤔 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Children always get the parent's process group when forked, so the chain would be preserved even in that case (I don't think it gets changed after the fact on There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is pretty in the weeds but I guess it does protect against callers of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's a good point. 👍 |
||
p, findErr := os.FindProcess(pid) | ||
if findErr != nil { | ||
d.logger.Trace("failed to find process of pid to kill", "pid", pid, "error", findErr) | ||
continue | ||
} | ||
processes = append(processes, p) | ||
if killErr := p.Kill(); killErr != nil { | ||
d.logger.Trace("failed to kill process", "pid", pid, "error", killErr) | ||
continue | ||
} | ||
} | ||
|
||
// thawed the cgroup so we can wait on each process | ||
thaw() | ||
|
||
// wait on each process | ||
for _, p := range processes { | ||
// do not capture error; errors are normal here | ||
pState, _ := p.Wait() | ||
d.logger.Trace("return from wait on process", "pid", p.Pid, "state", pState) | ||
} | ||
|
||
// cgroups are not atomic, the OS takes a moment to un-mark the cgroup as in-use; | ||
// a tiny sleep here goes a long way for not creating noisy (but functionally benign) | ||
// errors about removing busy cgroup | ||
// | ||
// alternatively we could do the removal in a loop and silence the interim errors, but meh | ||
time.Sleep(50 * time.Millisecond) | ||
|
||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
package resources | ||
|
||
// A Containment will cleanup resources created by an executor. | ||
type Containment interface { | ||
// Apply enables containment on pid. | ||
Apply(pid int) error | ||
|
||
// Cleanup will purge executor resources like cgroups. | ||
Cleanup() error | ||
|
||
// GetPIDs will return the processes overseen by the Containment | ||
GetPIDs() PIDs | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
//go:build !linux | ||
|
||
package resources | ||
|
||
type containment struct { | ||
// non-linux executors currently do not create resources to be cleaned up | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm having a little trouble following what we're doing with the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good point, the only implementation applies to Linux and the creation / usage is all gated in build-tagged files. The implementation in |
||
|
||
func (c *containment) Cleanup() error { | ||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
//go:build linux | ||
|
||
package resources | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
"path/filepath" | ||
"sync" | ||
|
||
"github.com/hashicorp/go-hclog" | ||
"github.com/hashicorp/nomad/client/lib/cgutil" | ||
"github.com/opencontainers/runc/libcontainer/cgroups" | ||
"github.com/opencontainers/runc/libcontainer/cgroups/fs2" | ||
"github.com/opencontainers/runc/libcontainer/configs" | ||
) | ||
|
||
type containment struct { | ||
lock sync.RWMutex | ||
cgroup *configs.Cgroup | ||
logger hclog.Logger | ||
} | ||
|
||
func Contain(logger hclog.Logger, cgroup *configs.Cgroup) *containment { | ||
return &containment{ | ||
cgroup: cgroup, | ||
logger: logger.Named("containment"), | ||
} | ||
} | ||
|
||
func (c *containment) Apply(pid int) error { | ||
c.lock.Lock() | ||
defer c.lock.Unlock() | ||
|
||
c.logger.Trace("create containment for", "cgroup", c.cgroup, "pid", pid) | ||
|
||
// for v2 use manager to create and enter the cgroup | ||
if cgutil.UseV2 { | ||
mgr, err := fs2.NewManager(c.cgroup, "", false) | ||
if err != nil { | ||
return fmt.Errorf("failed to create v2 cgroup manager for containment: %w", err) | ||
} | ||
|
||
// add the pid to the cgroup | ||
if err = mgr.Apply(pid); err != nil { | ||
return fmt.Errorf("failed to apply v2 cgroup containment: %w", err) | ||
} | ||
|
||
// in v2 it is important to set the device resource configuration | ||
if err = mgr.Set(c.cgroup.Resources); err != nil { | ||
return fmt.Errorf("failed to set v2 cgroup resources: %w", err) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
// for v1 a random cgroup was created already; just enter it | ||
if err := cgroups.EnterPid(c.cgroup.Paths, pid); err != nil { | ||
return fmt.Errorf("failed to add pid to v1 cgroup: %w", err) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func (c *containment) Cleanup() error { | ||
c.lock.Lock() | ||
defer c.lock.Unlock() | ||
|
||
// the current pid is of the executor, who manages the task process cleanup | ||
executorPID := os.Getpid() | ||
c.logger.Trace("cleanup on", "cgroup", c.cgroup, "executor_pid", executorPID) | ||
|
||
// destroy the task processes | ||
destroyer := cgutil.NewGroupKiller(c.logger, executorPID) | ||
return destroyer.KillGroup(c.cgroup) | ||
} | ||
|
||
func (c *containment) GetPIDs() PIDs { | ||
c.lock.Lock() | ||
defer c.lock.Unlock() | ||
|
||
m := make(PIDs) | ||
if c.cgroup == nil { | ||
return m | ||
} | ||
|
||
// get the cgroup path under containment | ||
var path string | ||
if cgutil.UseV2 { | ||
path = filepath.Join(cgutil.CgroupRoot, c.cgroup.Path) | ||
} else { | ||
path = c.cgroup.Paths["freezer"] | ||
} | ||
|
||
// find the pids in the cgroup under containment | ||
pids, err := cgroups.GetAllPids(path) | ||
if err != nil { | ||
c.logger.Debug("failed to get pids", "cgroup", c.cgroup, "error", err) | ||
return m | ||
} | ||
|
||
for _, pid := range pids { | ||
m[pid] = NewPID(pid) | ||
} | ||
|
||
return m | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
TestFS_Log
checks the alloc becomescomplete
, which it doesn't because it gets blocked by the missing cgroup for cpuset unless running as root.