diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index e57366e64b99..9f6033528100 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -119,7 +119,7 @@ func (d *RawExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstr func (d *RawExecDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) { // If we are on linux, running as root, cgroups are mounted, and cgroups - // aren't disabled by the operate use cgroups for pid management. + // aren't disabled by the operator use cgroups for pid management. forceDisable := d.DriverContext.config.ReadBoolDefault(rawExecNoCgroupOption, false) if !forceDisable && runtime.GOOS == "linux" && syscall.Geteuid() == 0 && cgroupsMounted(d.DriverContext.node) { diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go index 0188f7dfdc7f..4446db8d98bf 100644 --- a/client/driver/raw_exec_test.go +++ b/client/driver/raw_exec_test.go @@ -5,14 +5,18 @@ import ( "context" "fmt" "io/ioutil" + "os" "path/filepath" "reflect" + "strconv" + "syscall" "testing" "time" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/env" cstructs "github.com/hashicorp/nomad/client/structs" + tu "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/testtask" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" @@ -261,6 +265,92 @@ func TestRawExecDriver_Start_Kill_Wait(t *testing.T) { } } +func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) { + tu.ExecCompatible(t) + t.Parallel() + pidFile := "pid" + task := &structs.Task{ + Name: "sleep", + Driver: "raw_exec", + Config: map[string]interface{}{ + "command": testtask.Path(), + "args": []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"}, + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, + Resources: basicResources, + User: "root", + } + testtask.SetTaskEnv(task) + + ctx := testDriverContexts(t, task) + ctx.DriverCtx.node.Attributes["unique.cgroup.mountpoint"] = "foo" // Enable cgroups + defer ctx.AllocDir.Destroy() + d := NewRawExecDriver(ctx.DriverCtx) + + if _, err := d.Prestart(ctx.ExecCtx, task); err != nil { + t.Fatalf("prestart err: %v", err) + } + resp, err := d.Start(ctx.ExecCtx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Find the process + var pidData []byte + testutil.WaitForResult(func() (bool, error) { + var err error + pidData, err = ioutil.ReadFile(filepath.Join(ctx.AllocDir.AllocDir, "sleep", pidFile)) + if err != nil { + return false, err + } + + return true, nil + }, func(err error) { + t.Fatalf("err: %v", err) + }) + + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("failed to convert pid: %v", err) + } + + // Check the pid is up + process, err := os.FindProcess(pid) + if err != nil { + t.Fatalf("failed to find process") + } + if err := process.Signal(syscall.Signal(0)); err != nil { + t.Fatalf("process doesn't exist: %v", err) + } + + go func() { + time.Sleep(1 * time.Second) + err := resp.Handle.Kill() + + // Can't rely on the ordering between wait and kill on travis... + if !testutil.IsTravis() && err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case res := <-resp.Handle.WaitCh(): + if res.Successful() { + t.Fatal("should err") + } + case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): + t.Fatalf("timeout") + } + + if err := process.Signal(syscall.Signal(0)); err == nil { + t.Fatalf("process should not exist: %v", pid) + } +} + func TestRawExecDriver_HandlerExec(t *testing.T) { t.Parallel() task := &structs.Task{ diff --git a/client/fingerprint/cgroup.go b/client/fingerprint/cgroup.go index 2e6c446379ac..6a4a4c69ce0f 100644 --- a/client/fingerprint/cgroup.go +++ b/client/fingerprint/cgroup.go @@ -1,5 +1,3 @@ -// +build linux - package fingerprint import ( diff --git a/client/fingerprint/cgroup_default.go b/client/fingerprint/cgroup_default.go new file mode 100644 index 000000000000..948fcb97f9b2 --- /dev/null +++ b/client/fingerprint/cgroup_default.go @@ -0,0 +1,15 @@ +// +build !linux + +package fingerprint + +import cstructs "github.com/hashicorp/nomad/client/structs" + +// FindCgroupMountpointDir is used to find the cgroup mount point on a Linux +// system. Here it is a no-op implemtation +func FindCgroupMountpointDir() (string, error) { + return "", nil +} + +func (f *CGroupFingerprint) Fingerprint(*cstructs.FingerprintRequest, *cstructs.FingerprintResponse) error { + return nil +} diff --git a/client/testutil/driver_compatible.go b/client/testutil/driver_compatible.go index 7d928459a533..6c3b9cab436e 100644 --- a/client/testutil/driver_compatible.go +++ b/client/testutil/driver_compatible.go @@ -6,6 +6,8 @@ import ( "sync" "syscall" "testing" + + "github.com/hashicorp/nomad/client/fingerprint" ) // RequireRoot skips tests unless running on a Unix as root. @@ -19,6 +21,7 @@ func ExecCompatible(t *testing.T) { if runtime.GOOS != "linux" || syscall.Geteuid() != 0 { t.Skip("Test only available running as root on linux") } + CgroupCompatible(t) } func JavaCompatible(t *testing.T) { @@ -39,6 +42,13 @@ func QemuCompatible(t *testing.T) { } } +func CgroupCompatible(t *testing.T) { + mount, err := fingerprint.FindCgroupMountpointDir() + if err != nil || mount == "" { + t.Skipf("Failed to find cgroup mount: %v %v", mount, err) + } +} + var rktExists bool var rktOnce sync.Once diff --git a/helper/testtask/testtask.go b/helper/testtask/testtask.go index 9aea5bb7aaf6..bf7bd9fd5f2e 100644 --- a/helper/testtask/testtask.go +++ b/helper/testtask/testtask.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "os" "os/exec" + "strconv" + "syscall" "time" "github.com/hashicorp/nomad/nomad/structs" @@ -103,6 +105,50 @@ func execute() { file := popArg() ioutil.WriteFile(file, []byte(msg), 0666) + case "pgrp": + // pgrp puts the pid in a new process group + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "expected process group number for pgrp") + os.Exit(1) + } + num := popArg() + grp, err := strconv.Atoi(num) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to convert process group number %q: %v\n", num, err) + os.Exit(1) + } + if err := syscall.Setpgid(0, grp); err != nil { + fmt.Fprintf(os.Stderr, "failed to set process group: %v\n", err) + os.Exit(1) + } + + case "fork/exec": + // fork/exec forks execs the helper process + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "expect pid file and remaining args to fork exec") + os.Exit(1) + } + + pidFile := popArg() + + cmd := exec.Command(Path(), args...) + SetCmdEnv(cmd) + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "failed to fork/exec: %v\n", err) + os.Exit(1) + } + + if err := ioutil.WriteFile(pidFile, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 777); err != nil { + fmt.Fprintf(os.Stderr, "failed to write pid file: %v\n", err) + os.Exit(1) + } + + if err := cmd.Wait(); err != nil { + fmt.Fprintf(os.Stderr, "wait failed: %v\n", err) + os.Exit(1) + } + return + default: fmt.Fprintln(os.Stderr, "unknown command:", cmd) os.Exit(1) diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md index 1e7647d30f91..0ec6c2a35d1d 100644 --- a/website/source/docs/drivers/raw_exec.html.md +++ b/website/source/docs/drivers/raw_exec.html.md @@ -89,6 +89,17 @@ client { } ``` +## Client Options + +* `driver.raw_exec.enable` - Specifies whether the driver should be enabled or + disabled. + +* `driver.raw_exec.no_cgroups` - Specifies whether the driver should not use + cgroups to manage the process group launched by the driver. By default, + cgroups are used to manage the process tree to ensure full cleanup of all + processes started by the task. The driver only uses cgroups when Nomad is + launched as root, on Linux and when cgroups are detected. + ## Client Attributes The `raw_exec` driver will set the following client attributes: