diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 36b61d6e2dce..aba0ee8b4d07 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -118,6 +118,11 @@ type ExecCommand struct { // ResourceLimits determines whether resource limits are enforced by the // executor. ResourceLimits bool + + // Cgroup marks whether we put the process in a cgroup. Setting this field + // doesn't enforce resource limits. To enforce limits, set ResourceLimits. + // Using the cgroup does allow more precise cleanup of processes. + BasicProcessCgroup bool } // ProcessState holds information about the state of a user process. @@ -497,7 +502,7 @@ func (e *UniversalExecutor) Exit() error { } // Prefer killing the process via the resource container. - if e.cmd.Process != nil && !e.command.ResourceLimits { + if e.cmd.Process != nil && !(e.command.ResourceLimits || e.command.BasicProcessCgroup) { proc, err := os.FindProcess(e.cmd.Process.Pid) if err != nil { e.logger.Printf("[ERR] executor: can't find process with pid: %v, err: %v", @@ -508,7 +513,7 @@ func (e *UniversalExecutor) Exit() error { } } - if e.command.ResourceLimits { + if e.command.ResourceLimits || e.command.BasicProcessCgroup { if err := e.resConCtx.executorCleanup(); err != nil { merr.Errors = append(merr.Errors, err) } diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index ef1de7ebcf62..da7e7a239ffd 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -12,7 +12,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/mitchellh/go-ps" - "github.com/opencontainers/runc/libcontainer/cgroups" cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" @@ -36,7 +35,7 @@ func (e *UniversalExecutor) configureIsolation() error { } } - if e.command.ResourceLimits { + if e.command.ResourceLimits || e.command.BasicProcessCgroup { if err := e.configureCgroups(e.ctx.Task.Resources); err != nil { return fmt.Errorf("error creating cgroups: %v", err) } @@ -46,7 +45,7 @@ func (e *UniversalExecutor) configureIsolation() error { // applyLimits puts a process in a pre-configured cgroup func (e *UniversalExecutor) applyLimits(pid int) error { - if !e.command.ResourceLimits { + if !(e.command.ResourceLimits || e.command.BasicProcessCgroup) { return nil } @@ -56,7 +55,60 @@ func (e *UniversalExecutor) applyLimits(pid int) error { e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err) return err } + e.resConCtx.cgPaths = manager.GetPaths() + + // Don't enter all the cgroups since we will inherit resources limits. Only + // use devices (required by libcontainer) and freezer. Freezer allows us to + // capture all pids and stop any fork/execs from happening while we are + // cleaning up. + if !e.command.ResourceLimits { + // Move the executor into the global cgroup so that the task specific + // cgroup can be destroyed. + nilGroup := &cgroupConfig.Cgroup{} + nilGroup.Path = "/" + nilGroup.Resources = e.resConCtx.groups.Resources + nilManager := getCgroupManager(nilGroup, nil) + err := nilManager.Apply(pid) + if err != nil { + return fmt.Errorf("failed to remove executor pid %d: %v", pid, err) + } + + // Grab the freezer and devices cgroup paths. We do this from the old + // manager after the executor pid has been applied since there is no + // other way to determine what the proper cgroup paths would be. + freezer := &cgroupFs.FreezerGroup{} + devices := &cgroupFs.DevicesGroup{} + freezerName, devicesName := freezer.Name(), devices.Name() + newPath := map[string]string{ + freezerName: e.resConCtx.cgPaths[freezerName], + devicesName: e.resConCtx.cgPaths[devicesName], + } + + // Clear the cgroups paths so that everything is properly cleaned except + // the groups we want our process to stay in. This will delete the + // directories from disk. + manager.Cgroups.Paths = nil + delete(manager.Paths, freezerName) + delete(manager.Paths, devicesName) + if err := manager.Destroy(); err != nil { + e.logger.Printf("[ERR] executor: failed to destroy original: %v", err) + return err + } + + // Update our context such that the new cgroup manager only is tracking + // the paths we care about now. + e.resConCtx.cgPaths = newPath + e.resConCtx.groups.Paths = newPath + + // Apply just the freezer and devices now + manager = getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths) + if err := manager.Apply(pid); err != nil { + e.logger.Printf("[ERR] executor: error applying pid to cgroup subset %v: %v", e.resConCtx.cgPaths, err) + return err + } + } + cgConfig := cgroupConfig.Config{Cgroups: e.resConCtx.groups} if err := manager.Set(&cgConfig); err != nil { e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err) @@ -76,9 +128,14 @@ func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error cgroupName := uuid.Generate() e.resConCtx.groups.Path = filepath.Join("/nomad", cgroupName) - // TODO: verify this is needed for things like network access + // Allow access to /dev/ e.resConCtx.groups.Resources.AllowAllDevices = true + // Use a cgroup but don't apply limits + if !e.command.ResourceLimits { + return nil + } + if resources.MemoryMB > 0 { // Total amount of memory allowed to consume e.resConCtx.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024) @@ -110,6 +167,9 @@ func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error // isolation we aggregate the resource utilization of all the pids launched by // the executor. func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { + // If we don't use full resource limits fallback to normal collection. It is + // not enough to be in the Cgroup since you must be in the memory, cpu, and + // cpuacct cgroup to gather the correct statistics. if !e.command.ResourceLimits { pidStats, err := e.pidStats() if err != nil { @@ -234,7 +294,7 @@ func (e *UniversalExecutor) configureChroot() error { // isolation and we scan the entire process table if the user is not using any // isolation func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) { - if e.command.ResourceLimits { + if e.command.ResourceLimits || e.command.BasicProcessCgroup { manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths) pids, err := manager.GetAllPids() if err != nil { @@ -324,6 +384,9 @@ func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, execu proc.Wait() } + // Clear the cgroups paths so that everything is properly cleaned + manager.Cgroups.Paths = nil + // Remove the cgroup. if err := manager.Destroy(); err != nil { multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err)) @@ -332,6 +395,6 @@ func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, execu } // getCgroupManager returns the correct libcontainer cgroup manager. -func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager { +func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) *cgroupFs.Manager { return &cgroupFs.Manager{Cgroups: groups, Paths: paths} } diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index 0cea2ea554e5..9f6033528100 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -7,8 +7,11 @@ import ( "log" "os" "path/filepath" + "runtime" + "syscall" "time" + multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/driver/env" @@ -22,8 +25,11 @@ import ( ) const ( - // The option that enables this driver in the Config.Options map. - rawExecConfigOption = "driver.raw_exec.enable" + // rawExecEnableOption is the option that enables this driver in the Config.Options map. + rawExecEnableOption = "driver.raw_exec.enable" + + // rawExecNoCgroupOption forces no cgroups. + rawExecNoCgroupOption = "driver.raw_exec.no_cgroups" // The key populated in Node Attributes to indicate presence of the Raw Exec // driver @@ -36,21 +42,26 @@ const ( type RawExecDriver struct { DriverContext fingerprint.StaticFingerprinter + + // useCgroup tracks whether we should use a cgroup to manage the process + // tree + useCgroup bool } // rawExecHandle is returned from Start/Open as a handle to the PID type rawExecHandle struct { - version string - pluginClient *plugin.Client - userPid int - executor executor.Executor - killTimeout time.Duration - maxKillTimeout time.Duration - logger *log.Logger - waitCh chan *dstructs.WaitResult - doneCh chan struct{} - taskEnv *env.TaskEnv - taskDir *allocdir.TaskDir + version string + pluginClient *plugin.Client + userPid int + executor executor.Executor + isolationConfig *dstructs.IsolationConfig + killTimeout time.Duration + maxKillTimeout time.Duration + logger *log.Logger + waitCh chan *dstructs.WaitResult + doneCh chan struct{} + taskEnv *env.TaskEnv + taskDir *allocdir.TaskDir } // NewRawExecDriver is used to create a new raw exec driver @@ -93,7 +104,7 @@ func (d *RawExecDriver) FSIsolation() cstructs.FSIsolation { func (d *RawExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error { // Check that the user has explicitly enabled this executor. - enabled := req.Config.ReadBoolDefault(rawExecConfigOption, false) + enabled := req.Config.ReadBoolDefault(rawExecEnableOption, false) if enabled || req.Config.DevMode { d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed") @@ -107,6 +118,14 @@ func (d *RawExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstr } func (d *RawExecDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) { + // If we are on linux, running as root, cgroups are mounted, and cgroups + // aren't disabled by the operator use cgroups for pid management. + forceDisable := d.DriverContext.config.ReadBoolDefault(rawExecNoCgroupOption, false) + if !forceDisable && runtime.GOOS == "linux" && + syscall.Geteuid() == 0 && cgroupsMounted(d.DriverContext.node) { + d.useCgroup = true + } + return nil, nil } @@ -150,10 +169,11 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (*StartRespo } execCmd := &executor.ExecCommand{ - Cmd: command, - Args: driverConfig.Args, - User: task.User, - TaskKillSignal: taskKillSignal, + Cmd: command, + Args: driverConfig.Args, + User: task.User, + TaskKillSignal: taskKillSignal, + BasicProcessCgroup: d.useCgroup, } ps, err := exec.LaunchCmd(execCmd) if err != nil { @@ -165,17 +185,18 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (*StartRespo // Return a driver handle maxKill := d.DriverContext.config.MaxKillTimeout h := &rawExecHandle{ - pluginClient: pluginClient, - executor: exec, - userPid: ps.Pid, - killTimeout: GetKillTimeout(task.KillTimeout, maxKill), - maxKillTimeout: maxKill, - version: d.config.Version.VersionNumber(), - logger: d.logger, - doneCh: make(chan struct{}), - waitCh: make(chan *dstructs.WaitResult, 1), - taskEnv: ctx.TaskEnv, - taskDir: ctx.TaskDir, + pluginClient: pluginClient, + executor: exec, + isolationConfig: ps.IsolationConfig, + userPid: ps.Pid, + killTimeout: GetKillTimeout(task.KillTimeout, maxKill), + maxKillTimeout: maxKill, + version: d.config.Version.VersionNumber(), + logger: d.logger, + doneCh: make(chan struct{}), + waitCh: make(chan *dstructs.WaitResult, 1), + taskEnv: ctx.TaskEnv, + taskDir: ctx.TaskDir, } go h.run() return &StartResponse{Handle: h}, nil @@ -184,11 +205,12 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (*StartRespo func (d *RawExecDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil } type rawExecId struct { - Version string - KillTimeout time.Duration - MaxKillTimeout time.Duration - UserPid int - PluginConfig *PluginReattachConfig + Version string + KillTimeout time.Duration + MaxKillTimeout time.Duration + UserPid int + PluginConfig *PluginReattachConfig + IsolationConfig *dstructs.IsolationConfig } func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { @@ -202,11 +224,19 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e } exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { + merrs := new(multierror.Error) + merrs.Errors = append(merrs.Errors, err) d.logger.Println("[ERR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid") if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { - d.logger.Printf("[ERR] driver.raw_exec: error destroying plugin and userpid: %v", e) + merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e)) + } + if id.IsolationConfig != nil { + ePid := pluginConfig.Reattach.Pid + if e := executor.ClientCleanup(id.IsolationConfig, ePid); e != nil { + merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying resource container failed: %v", e)) + } } - return nil, fmt.Errorf("error connecting to plugin: %v", err) + return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil()) } ver, _ := exec.Version() @@ -214,17 +244,18 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e // Return a driver handle h := &rawExecHandle{ - pluginClient: pluginClient, - executor: exec, - userPid: id.UserPid, - logger: d.logger, - killTimeout: id.KillTimeout, - maxKillTimeout: id.MaxKillTimeout, - version: id.Version, - doneCh: make(chan struct{}), - waitCh: make(chan *dstructs.WaitResult, 1), - taskEnv: ctx.TaskEnv, - taskDir: ctx.TaskDir, + pluginClient: pluginClient, + executor: exec, + userPid: id.UserPid, + isolationConfig: id.IsolationConfig, + logger: d.logger, + killTimeout: id.KillTimeout, + maxKillTimeout: id.MaxKillTimeout, + version: id.Version, + doneCh: make(chan struct{}), + waitCh: make(chan *dstructs.WaitResult, 1), + taskEnv: ctx.TaskEnv, + taskDir: ctx.TaskDir, } go h.run() return h, nil @@ -232,11 +263,12 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e func (h *rawExecHandle) ID() string { id := rawExecId{ - Version: h.version, - KillTimeout: h.killTimeout, - MaxKillTimeout: h.maxKillTimeout, - PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), - UserPid: h.userPid, + Version: h.version, + KillTimeout: h.killTimeout, + MaxKillTimeout: h.maxKillTimeout, + PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), + UserPid: h.userPid, + IsolationConfig: h.isolationConfig, } data, err := json.Marshal(id) @@ -298,8 +330,15 @@ func (h *rawExecHandle) run() { ps, werr := h.executor.Wait() close(h.doneCh) if ps.ExitCode == 0 && werr != nil { - if e := killProcess(h.userPid); e != nil { - h.logger.Printf("[ERR] driver.raw_exec: error killing user process: %v", e) + if h.isolationConfig != nil { + ePid := h.pluginClient.ReattachConfig().Pid + if e := executor.ClientCleanup(h.isolationConfig, ePid); e != nil { + h.logger.Printf("[ERR] driver.raw_exec: destroying resource container failed: %v", e) + } + } else { + if e := killProcess(h.userPid); e != nil { + h.logger.Printf("[ERR] driver.raw_exec: error killing user process: %v", e) + } } } diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go index 1cec71a29fe7..c47319b07f25 100644 --- a/client/driver/raw_exec_test.go +++ b/client/driver/raw_exec_test.go @@ -5,14 +5,18 @@ import ( "context" "fmt" "io/ioutil" + "os" "path/filepath" "reflect" + "strconv" + "syscall" "testing" "time" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/env" cstructs "github.com/hashicorp/nomad/client/structs" + tu "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/testtask" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" @@ -33,7 +37,7 @@ func TestRawExecDriver_Fingerprint(t *testing.T) { } // Disable raw exec. - cfg := &config.Config{Options: map[string]string{rawExecConfigOption: "false"}} + cfg := &config.Config{Options: map[string]string{rawExecEnableOption: "false"}} request := &cstructs.FingerprintRequest{Config: cfg, Node: node} var response cstructs.FingerprintResponse @@ -47,7 +51,7 @@ func TestRawExecDriver_Fingerprint(t *testing.T) { } // Enable raw exec. - request.Config.Options[rawExecConfigOption] = "true" + request.Config.Options[rawExecEnableOption] = "true" err = d.Fingerprint(request, &response) if err != nil { t.Fatalf("err: %v", err) @@ -261,6 +265,101 @@ func TestRawExecDriver_Start_Kill_Wait(t *testing.T) { } } +// This test creates a process tree such that without cgroups tracking the +// processes cleanup of the children would not be possible. Thus the test +// asserts that the processes get killed properly when using cgroups. +func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) { + tu.ExecCompatible(t) + t.Parallel() + pidFile := "pid" + task := &structs.Task{ + Name: "sleep", + Driver: "raw_exec", + Config: map[string]interface{}{ + "command": testtask.Path(), + "args": []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"}, + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, + Resources: basicResources, + User: "root", + } + testtask.SetTaskEnv(task) + + ctx := testDriverContexts(t, task) + ctx.DriverCtx.node.Attributes["unique.cgroup.mountpoint"] = "foo" // Enable cgroups + defer ctx.AllocDir.Destroy() + d := NewRawExecDriver(ctx.DriverCtx) + + if _, err := d.Prestart(ctx.ExecCtx, task); err != nil { + t.Fatalf("prestart err: %v", err) + } + resp, err := d.Start(ctx.ExecCtx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Find the process + var pidData []byte + testutil.WaitForResult(func() (bool, error) { + var err error + pidData, err = ioutil.ReadFile(filepath.Join(ctx.AllocDir.AllocDir, "sleep", pidFile)) + if err != nil { + return false, err + } + + return true, nil + }, func(err error) { + t.Fatalf("err: %v", err) + }) + + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("failed to convert pid: %v", err) + } + + // Check the pid is up + process, err := os.FindProcess(pid) + if err != nil { + t.Fatalf("failed to find process") + } + if err := process.Signal(syscall.Signal(0)); err != nil { + t.Fatalf("process doesn't exist: %v", err) + } + + go func() { + time.Sleep(1 * time.Second) + err := resp.Handle.Kill() + + // Can't rely on the ordering between wait and kill on travis... + if !testutil.IsTravis() && err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case res := <-resp.Handle.WaitCh(): + if res.Successful() { + t.Fatal("should err") + } + case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): + t.Fatalf("timeout") + } + + testutil.WaitForResult(func() (bool, error) { + if err := process.Signal(syscall.Signal(0)); err == nil { + return false, fmt.Errorf("process should not exist: %v", pid) + } + + return true, nil + }, func(err error) { + t.Fatalf("err: %v", err) + }) +} + func TestRawExecDriver_HandlerExec(t *testing.T) { t.Parallel() task := &structs.Task{ diff --git a/client/fingerprint/cgroup.go b/client/fingerprint/cgroup.go index 2e6c446379ac..6a4a4c69ce0f 100644 --- a/client/fingerprint/cgroup.go +++ b/client/fingerprint/cgroup.go @@ -1,5 +1,3 @@ -// +build linux - package fingerprint import ( diff --git a/client/fingerprint/cgroup_default.go b/client/fingerprint/cgroup_default.go new file mode 100644 index 000000000000..948fcb97f9b2 --- /dev/null +++ b/client/fingerprint/cgroup_default.go @@ -0,0 +1,15 @@ +// +build !linux + +package fingerprint + +import cstructs "github.com/hashicorp/nomad/client/structs" + +// FindCgroupMountpointDir is used to find the cgroup mount point on a Linux +// system. Here it is a no-op implemtation +func FindCgroupMountpointDir() (string, error) { + return "", nil +} + +func (f *CGroupFingerprint) Fingerprint(*cstructs.FingerprintRequest, *cstructs.FingerprintResponse) error { + return nil +} diff --git a/client/testutil/driver_compatible.go b/client/testutil/driver_compatible.go index 7d928459a533..6c3b9cab436e 100644 --- a/client/testutil/driver_compatible.go +++ b/client/testutil/driver_compatible.go @@ -6,6 +6,8 @@ import ( "sync" "syscall" "testing" + + "github.com/hashicorp/nomad/client/fingerprint" ) // RequireRoot skips tests unless running on a Unix as root. @@ -19,6 +21,7 @@ func ExecCompatible(t *testing.T) { if runtime.GOOS != "linux" || syscall.Geteuid() != 0 { t.Skip("Test only available running as root on linux") } + CgroupCompatible(t) } func JavaCompatible(t *testing.T) { @@ -39,6 +42,13 @@ func QemuCompatible(t *testing.T) { } } +func CgroupCompatible(t *testing.T) { + mount, err := fingerprint.FindCgroupMountpointDir() + if err != nil || mount == "" { + t.Skipf("Failed to find cgroup mount: %v %v", mount, err) + } +} + var rktExists bool var rktOnce sync.Once diff --git a/helper/testtask/testtask.go b/helper/testtask/testtask.go index 9aea5bb7aaf6..bf7bd9fd5f2e 100644 --- a/helper/testtask/testtask.go +++ b/helper/testtask/testtask.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "os" "os/exec" + "strconv" + "syscall" "time" "github.com/hashicorp/nomad/nomad/structs" @@ -103,6 +105,50 @@ func execute() { file := popArg() ioutil.WriteFile(file, []byte(msg), 0666) + case "pgrp": + // pgrp puts the pid in a new process group + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "expected process group number for pgrp") + os.Exit(1) + } + num := popArg() + grp, err := strconv.Atoi(num) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to convert process group number %q: %v\n", num, err) + os.Exit(1) + } + if err := syscall.Setpgid(0, grp); err != nil { + fmt.Fprintf(os.Stderr, "failed to set process group: %v\n", err) + os.Exit(1) + } + + case "fork/exec": + // fork/exec forks execs the helper process + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "expect pid file and remaining args to fork exec") + os.Exit(1) + } + + pidFile := popArg() + + cmd := exec.Command(Path(), args...) + SetCmdEnv(cmd) + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "failed to fork/exec: %v\n", err) + os.Exit(1) + } + + if err := ioutil.WriteFile(pidFile, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 777); err != nil { + fmt.Fprintf(os.Stderr, "failed to write pid file: %v\n", err) + os.Exit(1) + } + + if err := cmd.Wait(); err != nil { + fmt.Fprintf(os.Stderr, "wait failed: %v\n", err) + os.Exit(1) + } + return + default: fmt.Fprintln(os.Stderr, "unknown command:", cmd) os.Exit(1) diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md index 1e7647d30f91..1d01bd90b90a 100644 --- a/website/source/docs/drivers/raw_exec.html.md +++ b/website/source/docs/drivers/raw_exec.html.md @@ -89,6 +89,17 @@ client { } ``` +## Client Options + +* `driver.raw_exec.enable` - Specifies whether the driver should be enabled or + disabled. + +* `driver.raw_exec.no_cgroups` - Specifies whether the driver should not use + cgroups to manage the process group launched by the driver. By default, + cgroups are used to manage the process tree to ensure full cleanup of all + processes started by the task. The driver only uses cgroups when Nomad is + launched as root, on Linux and when cgroups are detected. + ## Client Attributes The `raw_exec` driver will set the following client attributes: @@ -98,3 +109,10 @@ The `raw_exec` driver will set the following client attributes: ## Resource Isolation The `raw_exec` driver provides no isolation. + +If the launched process creates a new process group, it is possible that Nomad +will leak processes on shutdown unless the application forwards signals +properly. Nomad will not leak any processes if cgroups are being used to manage +the process tree. Cgroups are used on Linux when Nomad is being run with +appropriate priviledges, the cgroup system is mounted and the operator hasn't +disabled cgroups for the driver.