diff --git a/CHANGELOG.md b/CHANGELOG.md index db71c678a94f..c3c4bc2caefb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ FEATURES: IMPROVEMENTS: * cli: Improved `scaling policy` commands with -verbose, auto-completion, and prefix-matching [[GH-9964](https://github.com/hashicorp/nomad/issues/9964)] * consul/connect: Made handling of sidecar task container image URLs consistent with the `docker` task driver. [[GH-9580](https://github.com/hashicorp/nomad/issues/9580)] - * drivers/exec+java: Added client plugin configuration to re-enable previous PID/IPC namespace behavior [[GH-9982](https://github.com/hashicorp/nomad/pull/9982)] + * drivers/exec+java: Added client plugin and task configuration options to re-enable previous PID/IPC namespace behavior [[GH-9982](https://github.com/hashicorp/nomad/pull/9982)] [[tbd](tbd)] BUG FIXES: * consul: Fixed a bug where failing tasks with group services would only cause the allocation to restart once instead of respecting the `restart` field. [[GH-9869](https://github.com/hashicorp/nomad/issues/9869)] diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index f58b98e77453..1eb5d78a2483 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -78,8 +78,10 @@ var ( // taskConfigSpec is the hcl specification for the driver config section of // a task within a job. It is returned in the TaskConfigSchema RPC taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ - "command": hclspec.NewAttr("command", "string", true), - "args": hclspec.NewAttr("args", "list(string)", false), + "command": hclspec.NewAttr("command", "string", true), + "args": hclspec.NewAttr("args", "list(string)", false), + "pid_mode": hclspec.NewAttr("pid_mode", "string", false), + "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), }) // capabilities is returned by the Capabilities RPC and indicates what @@ -158,8 +160,35 @@ func (c *Config) validate() error { // TaskConfig is the driver configuration of a task within a job type TaskConfig struct { - Command string `codec:"command"` - Args []string `codec:"args"` + // Command is the thing to exec. + Command string `codec:"command"` + + // Args are passed along to Command. + Args []string `codec:"args"` + + // ModePID indicates whether PID namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModePID string `codec:"pid_mode"` + + // ModeIPC indicates whether IPC namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModeIPC string `codec:"ipc_mode"` +} + +func (tc *TaskConfig) validate() error { + switch tc.ModePID { + case "", executor.IsolationModePrivate, executor.IsolationModeHost: + default: + return fmt.Errorf("pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModePID) + } + + switch tc.ModeIPC { + case "", executor.IsolationModePrivate, executor.IsolationModeHost: + default: + return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) + } + + return nil } // TaskState is the state which is encoded in the handle returned in @@ -374,6 +403,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) } + if err := driverConfig.validate(); err != nil { + return nil, nil, fmt.Errorf("failed driver config validation: %v", err) + } + d.logger.Info("starting task", "driver_cfg", hclog.Fmt("%+v", driverConfig)) handle := drivers.NewTaskHandle(taskHandleVersion) handle.Config = cfg @@ -419,8 +452,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive Mounts: cfg.Mounts, Devices: cfg.Devices, NetworkIsolation: cfg.NetworkIsolation, - DefaultModePID: d.config.DefaultModePID, - DefaultModeIPC: d.config.DefaultModeIPC, + ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), + ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), } ps, err := exec.Launch(execCmd) diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index 5c74949703fe..be5dda5cf4c8 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -781,3 +781,25 @@ func TestDriver_Config_validate(t *testing.T) { }).validate()) } } + +func TestDriver_TaskConfig_validate(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } +} diff --git a/drivers/java/driver.go b/drivers/java/driver.go index a16d12bb93c5..51f7c8f545d2 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -85,6 +85,8 @@ var ( "jar_path": hclspec.NewAttr("jar_path", "string", false), "jvm_options": hclspec.NewAttr("jvm_options", "list(string)", false), "args": hclspec.NewAttr("args", "list(string)", false), + "pid_mode": hclspec.NewAttr("pid_mode", "string", false), + "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), }) // capabilities is returned by the Capabilities RPC and indicates what @@ -144,6 +146,25 @@ type TaskConfig struct { JarPath string `codec:"jar_path"` JvmOpts []string `codec:"jvm_options"` Args []string `codec:"args"` // extra arguments to java executable + ModePID string `codec:"pid_mode"` + ModeIPC string `codec:"ipc_mode"` +} + +func (tc *TaskConfig) validate() error { + switch tc.ModePID { + case "", executor.IsolationModePrivate, executor.IsolationModeHost: + default: + return fmt.Errorf("pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModePID) + + } + + switch tc.ModeIPC { + case "", executor.IsolationModePrivate, executor.IsolationModeHost: + default: + return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) + } + + return nil } // TaskState is the state which is encoded in the handle returned in @@ -369,6 +390,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) } + if err := driverConfig.validate(); err != nil { + return nil, nil, fmt.Errorf("failed driver config validation: %v", err) + } + if driverConfig.Class == "" && driverConfig.JarPath == "" { return nil, nil, fmt.Errorf("jar_path or class must be specified") } @@ -425,8 +450,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive Mounts: cfg.Mounts, Devices: cfg.Devices, NetworkIsolation: cfg.NetworkIsolation, - DefaultModePID: d.config.DefaultModePID, - DefaultModeIPC: d.config.DefaultModeIPC, + ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), + ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), } ps, err := exec.Launch(execCmd) diff --git a/drivers/shared/executor/client.go b/drivers/shared/executor/client.go index 7a2d9d9966a5..67724cc83e19 100644 --- a/drivers/shared/executor/client.go +++ b/drivers/shared/executor/client.go @@ -45,8 +45,8 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) { Mounts: drivers.MountsToProto(cmd.Mounts), Devices: drivers.DevicesToProto(cmd.Devices), NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation), - DefaultPidMode: cmd.DefaultModePID, - DefaultIpcMode: cmd.DefaultModeIPC, + DefaultPidMode: cmd.ModePID, + DefaultIpcMode: cmd.ModeIPC, } resp, err := c.client.Launch(ctx, req) if err != nil { diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index ce7c28dc0a46..bdee704a992f 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -141,11 +141,11 @@ type ExecCommand struct { // NetworkIsolation is the network isolation configuration. NetworkIsolation *drivers.NetworkIsolationSpec - // DefaultModePID is the default PID isolation mode - DefaultModePID string + // ModePID is the PID isolation mode (private or host). + ModePID string - // DefaultModeIPC is the default IPC isolation mode - DefaultModeIPC string + // ModeIPC is the IPC isolation mode (private or host). + ModeIPC string } // SetWriters sets the writer for the process stdout and stderr. This should diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index a8d1a6d5f6c6..3d5be7e80a42 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -590,7 +590,7 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error { cfg.NoPivotRoot = command.NoPivotRoot // set up default namespaces as configured - cfg.Namespaces = configureNamespaces(command.DefaultModePID, command.DefaultModeIPC) + cfg.Namespaces = configureNamespaces(command.ModePID, command.ModeIPC) if command.NetworkIsolation != nil { cfg.Namespaces = append(cfg.Namespaces, lconfigs.Namespace{ diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index fa484c216fdc..f6b038ef5e1f 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -129,8 +129,8 @@ func TestExecutor_Isolation_PID_and_IPC_hostMode(t *testing.T) { defer allocDir.Destroy() execCmd.ResourceLimits = true - execCmd.DefaultModePID = "host" // disable PID namespace - execCmd.DefaultModeIPC = "host" // disable IPC namespace + execCmd.ModePID = "host" // disable PID namespace + execCmd.ModeIPC = "host" // disable IPC namespace executor := NewExecutorWithIsolation(testlog.HCLogger(t)) defer executor.Shutdown("SIGKILL", 0) @@ -170,8 +170,8 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) { defer allocDir.Destroy() execCmd.ResourceLimits = true - execCmd.DefaultModePID = "private" - execCmd.DefaultModeIPC = "private" + execCmd.ModePID = "private" + execCmd.ModeIPC = "private" executor := NewExecutorWithIsolation(testlog.HCLogger(t)) defer executor.Shutdown("SIGKILL", 0) diff --git a/drivers/shared/executor/server.go b/drivers/shared/executor/server.go index bd8d4d7919e5..7ef91f4d341f 100644 --- a/drivers/shared/executor/server.go +++ b/drivers/shared/executor/server.go @@ -35,8 +35,8 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques Mounts: drivers.MountsFromProto(req.Mounts), Devices: drivers.DevicesFromProto(req.Devices), NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation), - DefaultModePID: req.DefaultPidMode, - DefaultModeIPC: req.DefaultIpcMode, + ModePID: req.DefaultPidMode, + ModeIPC: req.DefaultIpcMode, }) if err != nil { diff --git a/drivers/shared/executor/utils.go b/drivers/shared/executor/utils.go index 565cc80ac148..1e3cb74b9754 100644 --- a/drivers/shared/executor/utils.go +++ b/drivers/shared/executor/utils.go @@ -139,3 +139,13 @@ func processStateFromProto(pb *proto.ProcessState) (*ProcessState, error) { Time: timestamp, }, nil } + +// IsolationMode returns the namespace isolation mode as determined from agent +// plugin configuration and task driver configuration. The task configuration +// takes precedence, if it is configured. +func IsolationMode(plugin, task string) string { + if task != "" { + return task + } + return plugin +} diff --git a/drivers/shared/executor/utils_test.go b/drivers/shared/executor/utils_test.go new file mode 100644 index 000000000000..a8f12b5f3f46 --- /dev/null +++ b/drivers/shared/executor/utils_test.go @@ -0,0 +1,28 @@ +package executor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUtils_IsolationMode(t *testing.T) { + private := IsolationModePrivate + host := IsolationModeHost + blank := "" + + for _, tc := range []struct { + plugin, task, exp string + }{ + {plugin: private, task: private, exp: private}, + {plugin: private, task: host, exp: host}, + {plugin: private, task: blank, exp: private}, // default to private + + {plugin: host, task: private, exp: private}, + {plugin: host, task: host, exp: host}, + {plugin: host, task: blank, exp: host}, // default to host + } { + result := IsolationMode(tc.plugin, tc.task) + require.Equal(t, tc.exp, result) + } +} diff --git a/e2e/isolation/input/exec_host.nomad b/e2e/isolation/input/exec_host.nomad new file mode 100644 index 000000000000..984f3167158d --- /dev/null +++ b/e2e/isolation/input/exec_host.nomad @@ -0,0 +1,40 @@ +job "exec" { + datacenters = ["dc1"] + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "exec" { + task "exec" { + driver = "exec" + + config { + command = "bash" + args = [ + "-c", "local/pid.sh" + ] + pid_mode = "host" + ipc_mode = "host" + } + + template { + data = < **Warning:** If set to `"host"`, other processes running as the same user will + be able to access sensitive process information like environment variables. + +- `ipc_mode` - (Optional) Set to `"private"` to enable IPC namespace isolation for + this task, or `"host"` to disable isolation. If left unset, the behavior is + determined from the [`default_ipc_mode`][default_ipc_mode] in plugin configuration. + +!> **Warning:** If set to `"host"`, other processes running as the same user will be + able to make use of IPC features, like sending unexpected POSIX signals. + ## Examples To run a binary present on the Node: @@ -184,3 +198,6 @@ create. This list is configurable through the agent client [configuration file](/docs/configuration/client#chroot_env). + +[default_pid_mode]: /docs/drivers/exec#default_pid_mode +[default_ipc_mode]: /docs/drivers/exec#default_ipc_mode diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index 53bf8d2f6201..e3f0ac80a839 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -48,6 +48,20 @@ The `java` driver supports the following configuration in the job spec: - `jvm_options` - (Optional) A list of JVM options to be passed while invoking java. These options are passed without being validated in any way by Nomad. +- `pid_mode` - (Optional) Set to `"private"` to enable PID namespace isolation for + this task, or `"host"` to disable isolation. If left unset, the behavior is + determined from the [`default_pid_mode`][default_pid_mode] in plugin configuration. + +!> **Warning:** If set to `"host"`, other processes running as the same user will + be able to access sensitive process information like environment variables. + +- `ipc_mode` - (Optional) Set to `"private"` to enable IPC namespace isolation for + this task, or `"host"` to disable isolation. If left unset, the behavior is + determined from the [`default_ipc_mode`][default_ipc_mode] in plugin configuration. + +!> **Warning:** If set to `"host"`, other processes running as the same user will be + able to make use of IPC features, like sending unexpected POSIX signals. + ## Examples A simple config block to run a Java Jar: @@ -192,3 +206,6 @@ create. This list is configurable through the agent client [configuration file](/docs/configuration/client#chroot_env). + +[default_pid_mode]: /docs/drivers/java#default_pid_mode +[default_ipc_mode]: /docs/drivers/java#default_ipc_mode