From 191144c3bf7cdd504b0cbd0e6f00139e6c97f346 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 13 May 2021 14:21:52 -0600 Subject: [PATCH 1/8] drivers/exec: enable setting allow_caps on exec driver This PR enables setting allow_caps on the exec driver plugin configuration, as well as cap_add and cap_drop in exec task configuration. These options replicate the functionality already present in the docker task driver. Important: this change also reduces the default set of capabilities enabled by the exec driver to match the default set enabled by the docker driver. Until v1.0.5 the exec task driver would enable all capabilities supported by the operating system. v1.0.5 removed NET_RAW from that list of default capabilities, but left may others which could potentially also be leveraged by compromised tasks. Important: the "root" user is still special cased when used with the exec driver. Older versions of Nomad enabled enabled all capabilities supported by the operating system for tasks set with the root user. To maintain compatibility with existing clusters we continue supporting this "feature", however we maintain support for the legacy set of capabilities rather than enabling all capabilities now supported on modern operating systems. --- drivers/docker/config.go | 13 +- drivers/exec/driver.go | 73 +++++++- drivers/exec/driver_test.go | 127 ++++++++++---- drivers/exec/driver_unix_test.go | 127 ++++++++++++++ drivers/shared/capabilities/defaults.go | 114 ++++++++++++ drivers/shared/capabilities/defaults_test.go | 23 +++ drivers/shared/capabilities/set.go | 110 ++++++++++++ drivers/shared/capabilities/set_test.go | 162 ++++++++++++++++++ drivers/shared/executor/executor.go | 3 + drivers/shared/executor/executor_linux.go | 34 ++-- .../shared/executor/executor_linux_test.go | 11 +- drivers/shared/executor/proto/executor.pb.go | 140 ++++++++------- drivers/shared/executor/proto/executor.proto | 1 + 13 files changed, 806 insertions(+), 132 deletions(-) create mode 100644 drivers/shared/capabilities/defaults.go create mode 100644 drivers/shared/capabilities/defaults_test.go create mode 100644 drivers/shared/capabilities/set.go create mode 100644 drivers/shared/capabilities/set_test.go diff --git a/drivers/docker/config.go b/drivers/docker/config.go index ec919360d41b..898d9172c870 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -10,6 +10,7 @@ import ( docker "github.com/fsouza/go-dockerclient" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" @@ -287,7 +288,7 @@ var ( "allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false), "allow_caps": hclspec.NewDefault( hclspec.NewAttr("allow_caps", "list(string)", false), - hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), ), "nvidia_runtime": hclspec.NewDefault( hclspec.NewAttr("nvidia_runtime", "string", false), @@ -427,9 +428,9 @@ var ( "work_dir": hclspec.NewAttr("work_dir", "string", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the docker task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationImage, @@ -788,8 +789,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// Capabilities is returned by the Capabilities RPC and indicates what optional +// features this driver supports. func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } var _ drivers.InternalCapabilitiesDriver = (*Driver)(nil) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index b7cfb0058dbe..4357965083cf 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -74,6 +75,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -83,11 +88,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the exec task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationChroot, @@ -141,6 +148,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -156,6 +167,11 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } @@ -174,6 +190,12 @@ type TaskConfig struct { // ModeIPC indicates whether IPC namespace isolation is enabled for the task. // Must be "private" or "host" if set. ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -189,6 +211,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -266,8 +298,33 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// getCaps computes the complete set of linux capabilities to enable for driver, +// which gets passed along to libcontainer. +func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) { + driverAllowed := capabilities.New(d.config.AllowCaps) + + // determine caps the task wants that are not allowed + taskCaps := capabilities.New(tc.CapAdd) + missing := driverAllowed.Difference(taskCaps) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // if task did not specify allowed caps, use nomad defaults minus task drops + if len(tc.CapAdd) == 0 { + driverAllowed.Remove(tc.CapDrop) + return driverAllowed.Slice(true), nil + } + + // otherwise task did specify allowed caps, enable exactly those + taskAdd := capabilities.New(tc.CapAdd) + return taskAdd.Slice(true), nil +} + +// Capabilities is returned by the Capabilities RPC and indicates what +// optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -439,6 +496,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := d.getCaps(&driverConfig) + if err != nil { + return nil, nil, err + } + execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, Args: driverConfig.Args, @@ -455,6 +517,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -482,7 +545,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err := handle.SetDriverState(&driverState); err != nil { d.logger.Error("failed to start task, error setting driver state", "error", err) - exec.Shutdown("", 0) + _ = exec.Shutdown("", 0) pluginClient.Kill() return nil, nil, fmt.Errorf("failed to set driver state: %v", err) } diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index be5dda5cf4c8..8651eda20d21 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -764,42 +764,99 @@ func TestExecDriver_NoPivotRoot(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) } func TestDriver_TaskConfig_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "host", ipcMode: "", exp: nil}, - {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, - - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "", ipcMode: "host", exp: nil}, - {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&TaskConfig{ - ModePID: tc.pidMode, - ModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/exec/driver_unix_test.go b/drivers/exec/driver_unix_test.go index f8d77a2d7a11..3b2e60374aa8 100644 --- a/drivers/exec/driver_unix_test.go +++ b/drivers/exec/driver_unix_test.go @@ -5,9 +5,13 @@ package exec import ( "context" "fmt" + "strings" "testing" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" + "github.com/hashicorp/nomad/drivers/shared/executor" + basePlug "github.com/hashicorp/nomad/plugins/base" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -173,5 +177,128 @@ func TestExec_dnsConfig(t *testing.T) { dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) } +} + +func TestExecDriver_Capabilities(t *testing.T) { + ctestutils.ExecCompatible(t) + + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "sleep", + } + for _, tc := range []struct { + Name string + CapAdd []string + CapDrop []string + AllowList string + StartError string + }{ + { + Name: "default-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + }, + { + Name: "default-allowlist-add-forbidden", + CapAdd: []string{"net_admin"}, + StartError: "net_admin", + }, + { + Name: "default-allowlist-drop-existing", + CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + }, + { + Name: "restrictive-allowlist-drop-all", + CapDrop: []string{"ALL"}, + AllowList: "FOWNER,MKNOD", + }, + { + Name: "restrictive-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + }, + { + Name: "restrictive-allowlist-add-forbidden", + CapAdd: []string{"net_admin", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin", + }, + { + Name: "restrictive-allowlist-add-multiple-forbidden", + CapAdd: []string{"net_admin", "mknod", "CAP_SYS_TIME"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin, sys_time", + }, + { + Name: "permissive-allowlist", + CapAdd: []string{"net_admin", "mknod"}, + AllowList: "ALL", + }, + { + Name: "permissive-allowlist-add-all", + CapAdd: []string{"all"}, + AllowList: "ALL", + }, + } { + t.Run(tc.Name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + d := NewExecDriver(ctx, testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + config := &Config{ + NoPivotRoot: true, + DefaultModePID: executor.IsolationModePrivate, + DefaultModeIPC: executor.IsolationModePrivate, + } + + if tc.AllowList != "" { + config.AllowCaps = strings.Split(tc.AllowList, ",") + } else { + // inherit HCL defaults if not set + config.AllowCaps = capabilities.NomadDefaults().Slice(true) + } + + var data []byte + require.NoError(t, basePlug.MsgPackEncode(&data, config)) + baseConfig := &basePlug.Config{PluginConfig: data} + require.NoError(t, harness.SetConfig(baseConfig)) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + tCfg := &TaskConfig{ + Command: "/bin/sleep", + Args: []string{"9000"}, + } + if len(tc.CapAdd) > 0 { + tCfg.CapAdd = tc.CapAdd + } + if len(tc.CapDrop) > 0 { + tCfg.CapDrop = tc.CapDrop + } + require.NoError(t, task.EncodeConcreteDriverConfig(&tCfg)) + + // check the start error against expectations + _, _, err := harness.StartTask(task) + if err == nil && tc.StartError != "" { + t.Fatalf("Expected error in start: %v", tc.StartError) + } else if err != nil { + if tc.StartError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), tc.StartError) + } + return + } + + _ = d.DestroyTask(task.ID, true) + }) + } } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go new file mode 100644 index 000000000000..a1a6e3531e0e --- /dev/null +++ b/drivers/shared/capabilities/defaults.go @@ -0,0 +1,114 @@ +package capabilities + +import ( + "regexp" + + "github.com/syndtr/gocapability/capability" +) + +const ( + // HCLSpecLiteral is an equivalent list to NomadDefaults, expressed as a literal + // HCL string for use in HCL config parsing. + HCLSpecLiteral = `["AUDIT_WRITE","CHOWN","DAC_OVERRIDE","FOWNER","FSETID","KILL","MKNOD","NET_BIND_SERVICE","SETFCAP","SETGID","SETPCAP","SETUID","SYS_CHROOT"]` +) + +var ( + extractLiteral = regexp.MustCompile(`("[\w]+)`) +) + +// NomadDefaults is the set of Linux capabilities that Nomad enables by +// default. This list originates from what Docker enabled by default, but then +// excludes NET_RAW for security reasons. +// +// This set is use in the as HCL configuration default, described by HCLSpecLiteral. +func NomadDefaults() *Set { + return New(extractLiteral.FindAllString(HCLSpecLiteral, -1)) +} + +// DockerDefaults is a list of Linux capabilities enabled by Docker by default +// and is used to compute the set of capabilities to add/drop given docker driver +// configuration. +// +// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +func DockerDefaults() *Set { + defaults := NomadDefaults() + defaults.Add("NET_RAW") + return defaults +} + +// Supported returns the set of capabilities supported by the operating system. +// +// Defers to a library generated from +// https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h +func Supported() *Set { + s := New(nil) + + last := capability.CAP_LAST_CAP + + // workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap + if last == capability.Cap(63) { + last = capability.CAP_BLOCK_SUSPEND + } + + // accumulate every capability supported by this system + for _, c := range capability.List() { + if c > last { + continue + } + s.Add(c.String()) + } + + return s +} + +// LegacySupported returns the historical set of capabilities used when a task is +// configured to run as root using the exec task driver. Older versions of Nomad +// always allowed the root user to make use of any capability. Now that the exec +// task driver supports configuring the allowed capabilities, operators are +// encouraged to explicitly opt-in to capabilities beyond this legacy set. We +// maintain the legacy list here, because previous versions of Nomad deferred to +// the capability.List library function, which adds new capabilities over time. +// +// https://github.com/hashicorp/nomad/blob/v1.0.4/vendor/github.com/syndtr/gocapability/capability/enum_gen.go#L88 +func LegacySupported() *Set { + return New([]string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_MAC_OVERRIDE", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND", + "CAP_AUDIT_READ", + }) +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go new file mode 100644 index 000000000000..2a07afa0ea23 --- /dev/null +++ b/drivers/shared/capabilities/defaults_test.go @@ -0,0 +1,23 @@ +package capabilities + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_NomadDefaults(t *testing.T) { + result := NomadDefaults() + require.Len(t, result.Slice(false), 13) + defaults := strings.ToLower(HCLSpecLiteral) + for _, c := range result.Slice(false) { + require.Contains(t, defaults, c) + } +} + +func TestSet_DockerDefaults(t *testing.T) { + result := DockerDefaults() + require.Len(t, result.Slice(false), 14) + require.Contains(t, result.String(), "net_raw") +} diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go new file mode 100644 index 000000000000..046573e2a409 --- /dev/null +++ b/drivers/shared/capabilities/set.go @@ -0,0 +1,110 @@ +// Package capabilities is used for managing sets of linux capabilities. +package capabilities + +import ( + "sort" + "strings" +) + +type nothing struct{} + +var null = nothing{} + +// Set represents a group linux capabilities, implementing some useful set +// operations, taking care of name normalization, and sentinel value expansions. +// +// Linux capabilities can be expressed in multiple ways when working with docker +// and/or libcontainer, along with Nomad. +// +// Capability names may be upper or lower case, and may or may not be prefixed +// with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all" +// and "ALL" to mean "all capabilities supported by the operating system". +type Set struct { + data map[string]nothing +} + +func New(caps []string) *Set { + m := make(map[string]nothing, len(caps)) + for _, c := range caps { + insert(m, c) + } + return &Set{data: m} +} + +// Add cap into s. +func (s *Set) Add(cap string) { + insert(s.data, cap) +} + +func insert(data map[string]nothing, cap string) { + switch name := normalize(cap); name { + case "": + case "all": + for k, v := range Supported().data { + data[k] = v + } + return + default: + data[name] = null + } +} + +// Remove caps from s. +func (s *Set) Remove(caps []string) { + for _, c := range caps { + name := normalize(c) + if name == "all" { + s.data = make(map[string]nothing) + return + } + delete(s.data, name) + } +} + +// Difference returns the Set of elements of b not in s. +func (s *Set) Difference(b *Set) *Set { + data := make(map[string]nothing) + for c := range b.data { + if _, exists := s.data[c]; !exists { + data[c] = null + } + } + return &Set{data: data} +} + +// Empty return true if no capabilities exist in s. +func (s *Set) Empty() bool { + return len(s.data) == 0 +} + +// String returns the normalized and sorted string representation of s. +func (s *Set) String() string { + return strings.Join(s.Slice(false), ", ") +} + +// Slice returns a sorted slice of capabilities in s. +// +// big - indicates whether to uppercase and prefix capabilities with CAP_ +func (s *Set) Slice(upper bool) []string { + caps := make([]string, 0, len(s.data)) + for c := range s.data { + if upper { + c = "CAP_" + strings.ToUpper(c) + } + caps = append(caps, c) + } + sort.Strings(caps) + return caps +} + +// linux capabilities are often named in 4 possible ways - upper or lower case, +// and with or without a CAP_ prefix +// +// since we must do comparisons on cap names, always normalize the names before +// letting them into the Set data-structure +func normalize(name string) string { + spaces := strings.TrimSpace(name) + lower := strings.ToLower(spaces) + trim := strings.TrimPrefix(lower, "cap_") + return trim +} diff --git a/drivers/shared/capabilities/set_test.go b/drivers/shared/capabilities/set_test.go new file mode 100644 index 000000000000..8e072e8b0f28 --- /dev/null +++ b/drivers/shared/capabilities/set_test.go @@ -0,0 +1,162 @@ +package capabilities + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_Empty(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + result := New(nil).Empty() + require.True(t, result) + }) + + t.Run("empty", func(t *testing.T) { + result := New([]string{}).Empty() + require.True(t, result) + }) + + t.Run("full", func(t *testing.T) { + result := New([]string{"chown", "sys_time"}).Empty() + require.False(t, result) + }) +} + +func TestSet_New(t *testing.T) { + t.Parallel() + + t.Run("duplicates", func(t *testing.T) { + result := New([]string{"chown", "sys_time", "chown"}) + require.Equal(t, "chown, sys_time", result.String()) + }) + + t.Run("empty string", func(t *testing.T) { + result := New([]string{""}) + require.True(t, result.Empty()) + }) + + t.Run("all", func(t *testing.T) { + result := New([]string{"all"}) + exp := len(Supported().Slice(false)) + require.Len(t, result.Slice(false), exp) + }) +} + +func TestSet_Slice(t *testing.T) { + t.Parallel() + + exp := []string{"chown", "net_raw", "sys_time"} + + t.Run("lower case", func(t *testing.T) { + s := New([]string{"net_raw", "chown", "sys_time"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("upper case", func(t *testing.T) { + s := New([]string{"NET_RAW", "CHOWN", "SYS_TIME"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("prefix", func(t *testing.T) { + s := New([]string{"CAP_net_raw", "sys_TIME", "cap_chown"}) + require.Equal(t, exp, s.Slice(false)) + }) +} + +func TestSet_String(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + result := New(nil).String() + require.Equal(t, "", result) + }) + + t.Run("full", func(t *testing.T) { + exp := "chown, net_raw, sys_time" + in := []string{"net_raw", "CAP_CHOWN", "cap_sys_time"} + result := New(in).String() + require.Equal(t, exp, result) + }) +} + +func TestSet_Add(t *testing.T) { + t.Parallel() + + t.Run("add one", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + s.Add("CAP_SYS_TIME") + require.Equal(t, "chown, net_raw, sys_time", s.String()) + + s.Add("AF_NET") + require.Equal(t, "af_net, chown, net_raw, sys_time", s.String()) + }) + + t.Run("add empty string", func(t *testing.T) { + s := New([]string{"chown"}) + s.Add("") + require.Equal(t, "chown", s.String()) + }) + + t.Run("add all", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + exp := len(Supported().Slice(false)) + s.Add("all") + require.Len(t, s.Slice(false), exp) + }) + +} + +func TestSet_Remove(t *testing.T) { + t.Parallel() + + t.Run("remove one", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW"}) + require.Equal(t, "af_net, chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove couple", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW", "af_net"}) + require.Equal(t, "chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove all", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"all"}) + require.True(t, s.Empty()) + require.Equal(t, "", s.String()) + }) +} + +func TestSet_Difference(t *testing.T) { + t.Parallel() + + t.Run("a is empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"chown", "af_net"}) + result := a.Difference(b) + require.Equal(t, "af_net, chown", result.String()) + }) + + t.Run("b is empty", func(t *testing.T) { + a := New([]string{"chown", "af_net"}) + b := New(nil) + result := a.Difference(b) + require.True(t, result.Empty()) + }) + + t.Run("a diff b", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f"}) + b := New([]string{"B", "x", "Y", "a"}) + result := a.Difference(b) + require.Equal(t, "x, y", result.String()) + }) +} diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index a87674d2ae5c..7d738e45f638 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -147,6 +147,9 @@ type ExecCommand struct { // ModeIPC is the IPC isolation mode (private or host). ModeIPC string + + // Capabilities are the linux capabilities to be enabled by the task driver. + Capabilities []string } // SetWriters sets the writer for the process stdout and stderr. This should diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 16eb27b4c0f2..1bbb2839cbb6 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/opencontainers/runtime-spec/specs-go" "github.com/armon/circbuf" @@ -532,28 +533,26 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc ch <- &waitResult{ps, err} } -func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { - // TODO(shoenig): allow better control of these - // use capabilities list as prior to adopting libcontainer in 0.9 +func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { - // match capabilities used in Nomad 0.8 - if command.User == "root" { - allCaps := SupportedCaps(true) + switch command.User { + case "root": + // when running as root, use the legacy set of system capabilities, so + // that we do not break existing nomad clusters using this "feature" + legacyCaps := capabilities.LegacySupported().Slice(true) cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, - Permitted: allCaps, - Effective: allCaps, + Bounding: legacyCaps, + Permitted: legacyCaps, + Effective: legacyCaps, Ambient: nil, Inheritable: nil, } - } else { - allCaps := SupportedCaps(false) + default: + // otherwise apply the plugin + task capability configuration cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, + Bounding: command.Capabilities, } } - - return nil } func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces { @@ -759,16 +758,17 @@ func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) { }, Version: "1.0.0", } + for _, device := range specconv.AllowedDevices { cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule) } - if err := configureCapabilities(cfg, command); err != nil { - return nil, err - } + configureCapabilities(cfg, command) + if err := configureIsolation(cfg, command); err != nil { return nil, err } + if err := configureCgroups(cfg, command); err != nil { return nil, err } diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index a8bfba0a4e47..b3576797ad47 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/plugins/drivers" @@ -478,7 +479,7 @@ func TestExecutor_Capabilities(t *testing.T) { CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 -CapBnd: 0000003fffffdfff +CapBnd: 00000000a80405fb CapAmb: 0000000000000000`, }, { @@ -494,7 +495,6 @@ CapAmb: 0000000000000000`, for _, c := range cases { t.Run(c.user, func(t *testing.T) { - require := require.New(t) testExecCmd := testExecutorCommandWithChroot(t) execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir @@ -504,12 +504,13 @@ CapAmb: 0000000000000000`, execCmd.ResourceLimits = true execCmd.Cmd = "/bin/bash" execCmd.Args = []string{"-c", "cat /proc/$$/status"} + execCmd.Capabilities = capabilities.NomadDefaults().Slice(true) executor := NewExecutorWithIsolation(testlog.HCLogger(t)) defer executor.Shutdown("SIGKILL", 0) _, err := executor.Launch(execCmd) - require.NoError(err) + require.NoError(t, err) ch := make(chan interface{}) go func() { @@ -521,7 +522,7 @@ CapAmb: 0000000000000000`, case <-ch: // all good case <-time.After(5 * time.Second): - require.Fail("timeout waiting for exec to shutdown") + require.Fail(t, "timeout waiting for exec to shutdown") } canonical := func(s string) string { @@ -538,7 +539,7 @@ CapAmb: 0000000000000000`, return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) } return true, nil - }, func(err error) { require.NoError(err) }) + }, func(err error) { require.NoError(t, err) }) }) } diff --git a/drivers/shared/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go index f6defb54a30f..bf6c980c2904 100644 --- a/drivers/shared/executor/proto/executor.pb.go +++ b/drivers/shared/executor/proto/executor.pb.go @@ -44,6 +44,7 @@ type LaunchRequest struct { DefaultPidMode string `protobuf:"bytes,15,opt,name=default_pid_mode,json=defaultPidMode,proto3" json:"default_pid_mode,omitempty"` DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"` CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"` + AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -193,6 +194,13 @@ func (m *LaunchRequest) GetCpusetCgroup() string { return "" } +func (m *LaunchRequest) GetAllowCaps() []string { + if m != nil { + return m.AllowCaps + } + return nil +} + type LaunchResponse struct { Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -858,71 +866,73 @@ func init() { } var fileDescriptor_66b85426380683f3 = []byte{ - // 1020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x5b, 0x8f, 0xdb, 0x44, - 0x14, 0xae, 0x37, 0x9b, 0xdb, 0x49, 0xb2, 0x49, 0x47, 0xa8, 0xb8, 0xe1, 0xa1, 0xc1, 0x48, 0x34, - 0x82, 0xe2, 0xac, 0xb6, 0x37, 0x24, 0x24, 0x8a, 0xd8, 0x2d, 0xa8, 0xd2, 0x76, 0x15, 0x39, 0x85, - 0x4a, 0x3c, 0x60, 0x5c, 0xcf, 0x34, 0x19, 0x6d, 0xe2, 0x31, 0x33, 0xe3, 0x74, 0x91, 0x90, 0x78, - 0xe2, 0x1f, 0x80, 0xc4, 0x5f, 0xe5, 0x0d, 0xcd, 0xcd, 0x9b, 0x6c, 0x4b, 0xe5, 0x14, 0xf1, 0x14, - 0xcf, 0xc9, 0xf7, 0x9d, 0xcb, 0x9c, 0x73, 0xbe, 0x81, 0x3b, 0x98, 0xd3, 0x35, 0xe1, 0x62, 0x22, - 0x16, 0x09, 0x27, 0x78, 0x42, 0x2e, 0x48, 0x5a, 0x48, 0xc6, 0x27, 0x39, 0x67, 0x92, 0x95, 0xc7, - 0x50, 0x1f, 0xd1, 0xc7, 0x8b, 0x44, 0x2c, 0x68, 0xca, 0x78, 0x1e, 0x66, 0x6c, 0x95, 0xe0, 0x30, - 0x5f, 0x16, 0x73, 0x9a, 0x89, 0x70, 0x1b, 0x37, 0xbc, 0x35, 0x67, 0x6c, 0xbe, 0x24, 0xc6, 0xc9, - 0x8b, 0xe2, 0xe5, 0x44, 0xd2, 0x15, 0x11, 0x32, 0x59, 0xe5, 0x16, 0x10, 0x58, 0xe2, 0xc4, 0x85, - 0x37, 0xe1, 0xcc, 0xc9, 0x60, 0x82, 0xbf, 0xeb, 0xd0, 0x3b, 0x4d, 0x8a, 0x2c, 0x5d, 0x44, 0xe4, - 0xe7, 0x82, 0x08, 0x89, 0x06, 0x50, 0x4b, 0x57, 0xd8, 0xf7, 0x46, 0xde, 0xb8, 0x1d, 0xa9, 0x4f, - 0x84, 0x60, 0x3f, 0xe1, 0x73, 0xe1, 0xef, 0x8d, 0x6a, 0xe3, 0x76, 0xa4, 0xbf, 0xd1, 0x19, 0xb4, - 0x39, 0x11, 0xac, 0xe0, 0x29, 0x11, 0x7e, 0x6d, 0xe4, 0x8d, 0x3b, 0x47, 0x87, 0xe1, 0xbf, 0x25, - 0x6e, 0xe3, 0x9b, 0x90, 0x61, 0xe4, 0x78, 0xd1, 0xa5, 0x0b, 0x74, 0x0b, 0x3a, 0x42, 0x62, 0x56, - 0xc8, 0x38, 0x4f, 0xe4, 0xc2, 0xdf, 0xd7, 0xd1, 0xc1, 0x98, 0xa6, 0x89, 0x5c, 0x58, 0x00, 0xe1, - 0xdc, 0x00, 0xea, 0x25, 0x80, 0x70, 0xae, 0x01, 0x03, 0xa8, 0x91, 0x6c, 0xed, 0x37, 0x74, 0x92, - 0xea, 0x53, 0xe5, 0x5d, 0x08, 0xc2, 0xfd, 0xa6, 0xc6, 0xea, 0x6f, 0x74, 0x13, 0x5a, 0x32, 0x11, - 0xe7, 0x31, 0xa6, 0xdc, 0x6f, 0x69, 0x7b, 0x53, 0x9d, 0x4f, 0x28, 0x47, 0xb7, 0xa1, 0xef, 0xf2, - 0x89, 0x97, 0x74, 0x45, 0xa5, 0xf0, 0xdb, 0x23, 0x6f, 0xdc, 0x8a, 0x0e, 0x9c, 0xf9, 0x54, 0x5b, - 0xd1, 0x21, 0xbc, 0xf7, 0x22, 0x11, 0x34, 0x8d, 0x73, 0xce, 0x52, 0x22, 0x44, 0x9c, 0xce, 0x39, - 0x2b, 0x72, 0x1f, 0x34, 0x1a, 0xe9, 0xff, 0xa6, 0xe6, 0xaf, 0x63, 0xfd, 0x0f, 0x3a, 0x81, 0xc6, - 0x8a, 0x15, 0x99, 0x14, 0x7e, 0x67, 0x54, 0x1b, 0x77, 0x8e, 0xee, 0x54, 0xbc, 0xaa, 0xa7, 0x8a, - 0x14, 0x59, 0x2e, 0xfa, 0x16, 0x9a, 0x98, 0xac, 0xa9, 0xba, 0xf1, 0xae, 0x76, 0xf3, 0x59, 0x45, - 0x37, 0x27, 0x9a, 0x15, 0x39, 0x36, 0x5a, 0xc0, 0xf5, 0x8c, 0xc8, 0x57, 0x8c, 0x9f, 0xc7, 0x54, - 0xb0, 0x65, 0x22, 0x29, 0xcb, 0xfc, 0x9e, 0x6e, 0xe2, 0x17, 0x15, 0x5d, 0x9e, 0x19, 0xfe, 0x13, - 0x47, 0x9f, 0xe5, 0x24, 0x8d, 0x06, 0xd9, 0x15, 0x2b, 0x0a, 0xa0, 0x97, 0xb1, 0x38, 0xa7, 0x6b, - 0x26, 0x63, 0xce, 0x98, 0xf4, 0x0f, 0xf4, 0x1d, 0x75, 0x32, 0x36, 0x55, 0xb6, 0x88, 0x31, 0x89, - 0xc6, 0x30, 0xc0, 0xe4, 0x65, 0x52, 0x2c, 0x65, 0x9c, 0x53, 0x1c, 0xaf, 0x18, 0x26, 0x7e, 0x5f, - 0xb7, 0xe6, 0xc0, 0xda, 0xa7, 0x14, 0x3f, 0x65, 0x98, 0x6c, 0x22, 0x69, 0x9e, 0x1a, 0xe4, 0x60, - 0x0b, 0xf9, 0x24, 0x4f, 0x35, 0xf2, 0x23, 0xe8, 0xa5, 0x79, 0x21, 0x88, 0x74, 0xbd, 0xb9, 0xae, - 0x61, 0x5d, 0x63, 0x34, 0x5d, 0x09, 0x7e, 0x82, 0x03, 0x37, 0xfa, 0x22, 0x67, 0x99, 0x20, 0xe8, - 0x0c, 0x9a, 0xb6, 0xa7, 0x7a, 0xfe, 0x3b, 0x47, 0xf7, 0xc2, 0x6a, 0xcb, 0x18, 0xda, 0x7e, 0xcf, - 0x64, 0x22, 0x49, 0xe4, 0x9c, 0x04, 0x3d, 0xe8, 0x3c, 0x4f, 0xa8, 0xb4, 0xab, 0x15, 0xfc, 0x08, - 0x5d, 0x73, 0xfc, 0x9f, 0xc2, 0x9d, 0x42, 0x7f, 0xb6, 0x28, 0x24, 0x66, 0xaf, 0x32, 0xb7, 0xcd, - 0x37, 0xa0, 0x21, 0xe8, 0x3c, 0x4b, 0x96, 0x76, 0xa1, 0xed, 0x09, 0x7d, 0x08, 0xdd, 0x39, 0x4f, - 0x52, 0x12, 0xe7, 0x84, 0x53, 0x86, 0xfd, 0xbd, 0x91, 0x37, 0xae, 0x45, 0x1d, 0x6d, 0x9b, 0x6a, - 0x53, 0x80, 0x60, 0x70, 0xe9, 0xcd, 0x64, 0x1c, 0x2c, 0xe0, 0xc6, 0x77, 0x39, 0x56, 0x41, 0xcb, - 0x25, 0xb6, 0x81, 0xb6, 0x04, 0xc1, 0xfb, 0xcf, 0x82, 0x10, 0xdc, 0x84, 0xf7, 0x5f, 0x8b, 0x64, - 0x93, 0x18, 0xc0, 0xc1, 0xf7, 0x84, 0x0b, 0xca, 0x5c, 0x95, 0xc1, 0xa7, 0xd0, 0x2f, 0x2d, 0xf6, - 0x6e, 0x7d, 0x68, 0xae, 0x8d, 0xc9, 0x56, 0xee, 0x8e, 0xc1, 0x27, 0xd0, 0x55, 0xf7, 0x56, 0x66, - 0x3e, 0x84, 0x16, 0xcd, 0x24, 0xe1, 0x6b, 0x7b, 0x49, 0xb5, 0xa8, 0x3c, 0x07, 0xcf, 0xa1, 0x67, - 0xb1, 0xd6, 0xed, 0x37, 0x50, 0x17, 0xca, 0xb0, 0x63, 0x89, 0xcf, 0x12, 0x71, 0x6e, 0x1c, 0x19, - 0x7a, 0x70, 0x1b, 0x7a, 0x33, 0xdd, 0x89, 0x37, 0x37, 0xaa, 0xee, 0x1a, 0xa5, 0x8a, 0x75, 0x40, - 0x5b, 0xfe, 0x39, 0x74, 0x1e, 0x5f, 0x90, 0xd4, 0x11, 0x1f, 0x40, 0x0b, 0x93, 0x04, 0x2f, 0x69, - 0x46, 0x6c, 0x52, 0xc3, 0xd0, 0xbc, 0x0c, 0xa1, 0x7b, 0x19, 0xc2, 0x67, 0xee, 0x65, 0x88, 0x4a, - 0xac, 0xd3, 0xf9, 0xbd, 0xd7, 0x75, 0xbe, 0x76, 0xa9, 0xf3, 0xc1, 0x31, 0x74, 0x4d, 0x30, 0x5b, - 0xff, 0x0d, 0x68, 0xb0, 0x42, 0xe6, 0x85, 0xd4, 0xb1, 0xba, 0x91, 0x3d, 0xa1, 0x0f, 0xa0, 0x4d, - 0x2e, 0xa8, 0x8c, 0x53, 0xb5, 0x93, 0x7b, 0xba, 0x82, 0x96, 0x32, 0x1c, 0x33, 0x4c, 0x82, 0xdf, - 0x3d, 0xe8, 0x6e, 0x4e, 0xac, 0x8a, 0x9d, 0x53, 0x6c, 0x2b, 0x55, 0x9f, 0x6f, 0xe5, 0x6f, 0xdc, - 0x4d, 0x6d, 0xf3, 0x6e, 0x50, 0x08, 0xfb, 0xea, 0xcd, 0xd3, 0xaf, 0xc5, 0xdb, 0xcb, 0xd6, 0xb8, - 0xa3, 0x3f, 0xdb, 0xd0, 0x7a, 0x6c, 0x17, 0x09, 0xfd, 0x02, 0x0d, 0xb3, 0xfd, 0xe8, 0x7e, 0xd5, - 0xad, 0xdb, 0x7a, 0x28, 0x87, 0x0f, 0x76, 0xa5, 0xd9, 0xfe, 0x5d, 0x43, 0x02, 0xf6, 0x95, 0x0e, - 0xa0, 0xbb, 0x55, 0x3d, 0x6c, 0x88, 0xc8, 0xf0, 0xde, 0x6e, 0xa4, 0x32, 0xe8, 0x6f, 0xd0, 0x72, - 0xeb, 0x8c, 0x1e, 0x56, 0xf5, 0x71, 0x45, 0x4e, 0x86, 0x9f, 0xef, 0x4e, 0x2c, 0x13, 0xf8, 0xc3, - 0x83, 0xfe, 0x95, 0x95, 0x46, 0x5f, 0x56, 0xf5, 0xf7, 0x66, 0xd5, 0x19, 0x3e, 0x7a, 0x67, 0x7e, - 0x99, 0xd6, 0xaf, 0xd0, 0xb4, 0xda, 0x81, 0x2a, 0x77, 0x74, 0x5b, 0x7e, 0x86, 0x0f, 0x77, 0xe6, - 0x95, 0xd1, 0x2f, 0xa0, 0xae, 0x75, 0x01, 0x55, 0x6e, 0xeb, 0xa6, 0x76, 0x0d, 0xef, 0xef, 0xc8, - 0x72, 0x71, 0x0f, 0x3d, 0x35, 0xff, 0x46, 0x58, 0xaa, 0xcf, 0xff, 0x96, 0x62, 0x55, 0x9f, 0xff, - 0x2b, 0xfa, 0xa5, 0xe7, 0x5f, 0xad, 0x61, 0xf5, 0xf9, 0xdf, 0xd0, 0xbb, 0xea, 0xf3, 0xbf, 0xa9, - 0x5b, 0xc1, 0x35, 0xf4, 0x97, 0x07, 0x3d, 0x65, 0x9a, 0x49, 0x4e, 0x92, 0x15, 0xcd, 0xe6, 0xe8, - 0x51, 0x45, 0xf1, 0x56, 0x2c, 0x23, 0xe0, 0x96, 0xe9, 0x52, 0xf9, 0xea, 0xdd, 0x1d, 0xb8, 0xb4, - 0xc6, 0xde, 0xa1, 0xf7, 0x75, 0xf3, 0x87, 0xba, 0xd1, 0xac, 0x86, 0xfe, 0xb9, 0xfb, 0x4f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x32, 0xbc, 0x95, 0x4b, 0x31, 0x0c, 0x00, 0x00, + // 1041 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xed, 0x6f, 0x1b, 0xc5, + 0x13, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0xae, 0x7e, 0xca, 0xef, 0x6a, 0x84, 0x6a, 0x0e, + 0x89, 0x5a, 0x50, 0x2e, 0x51, 0xfa, 0x86, 0x84, 0x44, 0x11, 0x49, 0x41, 0x95, 0xd2, 0x28, 0xba, + 0x14, 0x2a, 0xf1, 0x81, 0x63, 0x7b, 0xb7, 0xb5, 0x57, 0xb1, 0x6f, 0x97, 0xdd, 0x3d, 0x27, 0x48, + 0x48, 0x7c, 0xe2, 0x3f, 0x00, 0xa9, 0x7f, 0x2e, 0xda, 0xb7, 0x8b, 0x9d, 0x96, 0xea, 0x5c, 0xc4, + 0x27, 0xdf, 0x8e, 0x9f, 0x67, 0x66, 0x76, 0x67, 0xe6, 0x19, 0xb8, 0x9b, 0x0b, 0xba, 0x20, 0x42, + 0xee, 0xc9, 0x29, 0x16, 0x24, 0xdf, 0x23, 0x97, 0x24, 0x2b, 0x15, 0x13, 0x7b, 0x5c, 0x30, 0xc5, + 0xaa, 0x63, 0x6c, 0x8e, 0xe8, 0x93, 0x29, 0x96, 0x53, 0x9a, 0x31, 0xc1, 0xe3, 0x82, 0xcd, 0x71, + 0x1e, 0xf3, 0x59, 0x39, 0xa1, 0x85, 0x8c, 0x57, 0x71, 0xc3, 0xdb, 0x13, 0xc6, 0x26, 0x33, 0x62, + 0x9d, 0xbc, 0x2c, 0x5f, 0xed, 0x29, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0xf7, + 0x7c, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0x7a, 0xdd, 0x84, 0xfe, 0x31, 0x2e, 0x8b, 0x6c, 0x9a, + 0x90, 0x5f, 0x4a, 0x22, 0x15, 0x1a, 0x40, 0x23, 0x9b, 0xe7, 0x61, 0x30, 0x0a, 0xc6, 0x9d, 0x44, + 0x7f, 0x22, 0x04, 0x9b, 0x58, 0x4c, 0x64, 0xb8, 0x31, 0x6a, 0x8c, 0x3b, 0x89, 0xf9, 0x46, 0x27, + 0xd0, 0x11, 0x44, 0xb2, 0x52, 0x64, 0x44, 0x86, 0x8d, 0x51, 0x30, 0xee, 0x1e, 0xec, 0xc7, 0xff, + 0x94, 0xb8, 0x8b, 0x6f, 0x43, 0xc6, 0x89, 0xe7, 0x25, 0x57, 0x2e, 0xd0, 0x6d, 0xe8, 0x4a, 0x95, + 0xb3, 0x52, 0xa5, 0x1c, 0xab, 0x69, 0xb8, 0x69, 0xa2, 0x83, 0x35, 0x9d, 0x62, 0x35, 0x75, 0x00, + 0x22, 0x84, 0x05, 0x6c, 0x55, 0x00, 0x22, 0x84, 0x01, 0x0c, 0xa0, 0x41, 0x8a, 0x45, 0xd8, 0x34, + 0x49, 0xea, 0x4f, 0x9d, 0x77, 0x29, 0x89, 0x08, 0x5b, 0x06, 0x6b, 0xbe, 0xd1, 0x2d, 0x68, 0x2b, + 0x2c, 0xcf, 0xd3, 0x9c, 0x8a, 0xb0, 0x6d, 0xec, 0x2d, 0x7d, 0x3e, 0xa2, 0x02, 0xdd, 0x81, 0x1d, + 0x9f, 0x4f, 0x3a, 0xa3, 0x73, 0xaa, 0x64, 0xd8, 0x19, 0x05, 0xe3, 0x76, 0xb2, 0xed, 0xcd, 0xc7, + 0xc6, 0x8a, 0xf6, 0xe1, 0x7f, 0x2f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, + 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x9d, 0xda, 0xbf, 0x0e, 0xcd, 0x3f, 0xe8, 0x08, + 0x9a, 0x73, 0x56, 0x16, 0x4a, 0x86, 0xdd, 0x51, 0x63, 0xdc, 0x3d, 0xb8, 0x5b, 0xf3, 0xa9, 0x9e, + 0x69, 0x52, 0xe2, 0xb8, 0xe8, 0x3b, 0x68, 0xe5, 0x64, 0x41, 0xf5, 0x8b, 0xf7, 0x8c, 0x9b, 0xcf, + 0x6b, 0xba, 0x39, 0x32, 0xac, 0xc4, 0xb3, 0xd1, 0x14, 0x6e, 0x16, 0x44, 0x5d, 0x30, 0x71, 0x9e, + 0x52, 0xc9, 0x66, 0x58, 0x51, 0x56, 0x84, 0x7d, 0x53, 0xc4, 0x2f, 0x6b, 0xba, 0x3c, 0xb1, 0xfc, + 0xa7, 0x9e, 0x7e, 0xc6, 0x49, 0x96, 0x0c, 0x8a, 0x6b, 0x56, 0x14, 0x41, 0xbf, 0x60, 0x29, 0xa7, + 0x0b, 0xa6, 0x52, 0xc1, 0x98, 0x0a, 0xb7, 0xcd, 0x1b, 0x75, 0x0b, 0x76, 0xaa, 0x6d, 0x09, 0x63, + 0x0a, 0x8d, 0x61, 0x90, 0x93, 0x57, 0xb8, 0x9c, 0xa9, 0x94, 0xd3, 0x3c, 0x9d, 0xb3, 0x9c, 0x84, + 0x3b, 0xa6, 0x34, 0xdb, 0xce, 0x7e, 0x4a, 0xf3, 0x67, 0x2c, 0x27, 0xcb, 0x48, 0xca, 0x33, 0x8b, + 0x1c, 0xac, 0x20, 0x9f, 0xf2, 0xcc, 0x20, 0x3f, 0x86, 0x7e, 0xc6, 0x4b, 0x49, 0x94, 0xaf, 0xcd, + 0x4d, 0x03, 0xeb, 0x59, 0xa3, 0xab, 0xca, 0x87, 0x00, 0x78, 0x36, 0x63, 0x17, 0x69, 0x86, 0xb9, + 0x0c, 0x91, 0x69, 0x9c, 0x8e, 0xb1, 0x1c, 0x62, 0x2e, 0xa3, 0x9f, 0x61, 0xdb, 0x4f, 0x86, 0xe4, + 0xac, 0x90, 0x04, 0x9d, 0x40, 0xcb, 0x95, 0xdc, 0x8c, 0x47, 0xf7, 0xe0, 0x7e, 0x5c, 0x6f, 0x56, + 0x63, 0xd7, 0x0e, 0x67, 0x0a, 0x2b, 0x92, 0x78, 0x27, 0x51, 0x1f, 0xba, 0x2f, 0x30, 0x55, 0x6e, + 0xf2, 0xa2, 0x9f, 0xa0, 0x67, 0x8f, 0xff, 0x51, 0xb8, 0x63, 0xd8, 0x39, 0x9b, 0x96, 0x2a, 0x67, + 0x17, 0x85, 0x1f, 0xf6, 0x5d, 0x68, 0x4a, 0x3a, 0x29, 0xf0, 0xcc, 0xcd, 0xbb, 0x3b, 0xa1, 0x8f, + 0xa0, 0x37, 0x11, 0x38, 0x23, 0x29, 0x27, 0x82, 0xb2, 0x3c, 0xdc, 0x18, 0x05, 0xe3, 0x46, 0xd2, + 0x35, 0xb6, 0x53, 0x63, 0x8a, 0x10, 0x0c, 0xae, 0xbc, 0xd9, 0x8c, 0xa3, 0x29, 0xec, 0x7e, 0xcf, + 0x73, 0x1d, 0xb4, 0x9a, 0x71, 0x17, 0x68, 0x45, 0x2f, 0x82, 0x7f, 0xad, 0x17, 0xd1, 0x2d, 0xf8, + 0xff, 0x1b, 0x91, 0x5c, 0x12, 0x03, 0xd8, 0xfe, 0x81, 0x08, 0x49, 0x99, 0xbf, 0x65, 0xf4, 0x19, + 0xec, 0x54, 0x16, 0xf7, 0xb6, 0x21, 0xb4, 0x16, 0xd6, 0xe4, 0x6e, 0xee, 0x8f, 0xd1, 0xa7, 0xd0, + 0xd3, 0xef, 0x56, 0x65, 0x3e, 0x84, 0x36, 0x2d, 0x14, 0x11, 0x0b, 0xf7, 0x48, 0x8d, 0xa4, 0x3a, + 0x47, 0x2f, 0xa0, 0xef, 0xb0, 0xce, 0xed, 0xb7, 0xb0, 0x25, 0xb5, 0x61, 0xcd, 0x2b, 0x3e, 0xc7, + 0xf2, 0xdc, 0x3a, 0xb2, 0xf4, 0xe8, 0x0e, 0xf4, 0xcf, 0x4c, 0x25, 0xde, 0x5e, 0xa8, 0x2d, 0x5f, + 0x28, 0x7d, 0x59, 0x0f, 0x74, 0xd7, 0x3f, 0x87, 0xee, 0x93, 0x4b, 0x92, 0x79, 0xe2, 0x43, 0x68, + 0xe7, 0x04, 0xe7, 0x33, 0x5a, 0x10, 0x97, 0xd4, 0x30, 0xb6, 0x8b, 0x23, 0xf6, 0x8b, 0x23, 0x7e, + 0xee, 0x17, 0x47, 0x52, 0x61, 0xfd, 0x1a, 0xd8, 0x78, 0x73, 0x0d, 0x34, 0xae, 0xd6, 0x40, 0x74, + 0x08, 0x3d, 0x1b, 0xcc, 0xdd, 0x7f, 0x17, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, 0x71, + 0x27, 0xf4, 0x01, 0x74, 0xc8, 0x25, 0x55, 0x69, 0xa6, 0x47, 0x76, 0xc3, 0xdc, 0xa0, 0xad, 0x0d, + 0x87, 0x2c, 0x27, 0xd1, 0x1f, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, 0xd5, + 0x9f, 0xef, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x25, 0x9a, 0x65, + 0xf2, 0xee, 0x6b, 0x1b, 0xdc, 0xc1, 0x5f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x85, 0xa6, + 0x9d, 0x7e, 0xf4, 0xa0, 0xee, 0xd4, 0xad, 0xec, 0xd1, 0xe1, 0xc3, 0x75, 0x69, 0xae, 0x7e, 0x37, + 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x5e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xbf, 0x1e, 0xa9, + 0x0a, 0xfa, 0x3b, 0xb4, 0xfd, 0x38, 0xa3, 0x47, 0x75, 0x7d, 0x5c, 0x93, 0x93, 0xe1, 0x17, 0xeb, + 0x13, 0xab, 0x04, 0xfe, 0x0c, 0x60, 0xe7, 0xda, 0x48, 0xa3, 0xaf, 0xea, 0xfa, 0x7b, 0xbb, 0xea, + 0x0c, 0x1f, 0xbf, 0x37, 0xbf, 0x4a, 0xeb, 0x37, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, 0x2a, + 0x3f, 0xc3, 0x47, 0x6b, 0xf3, 0xaa, 0xe8, 0x97, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, 0xb2, + 0x76, 0x0d, 0x1f, 0xac, 0xc9, 0xf2, 0x71, 0xf7, 0x03, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, 0xff, + 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x6b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, 0xd2, + 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x0d, 0xf4, 0x3a, 0x80, 0xbe, 0x36, 0x9d, 0x29, 0x41, + 0xf0, 0x9c, 0x16, 0x13, 0xf4, 0xb8, 0xa6, 0x78, 0x6b, 0x96, 0x15, 0x70, 0xc7, 0xf4, 0xa9, 0x7c, + 0xfd, 0xfe, 0x0e, 0x7c, 0x5a, 0xe3, 0x60, 0x3f, 0xf8, 0xa6, 0xf5, 0xe3, 0x96, 0xd5, 0xac, 0xa6, + 0xf9, 0xb9, 0xf7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x41, 0x13, 0xe3, 0x8e, 0x50, 0x0c, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/drivers/shared/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto index db0a7b3cbfdc..9ac241209fcf 100644 --- a/drivers/shared/executor/proto/executor.proto +++ b/drivers/shared/executor/proto/executor.proto @@ -45,6 +45,7 @@ message LaunchRequest { string default_pid_mode = 15; string default_ipc_mode = 16; string cpuset_cgroup = 17; + repeated string allow_caps = 18; } message LaunchResponse { From 9bb4b8fa04c727eca6aad43d9baa85f3271e143f Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 10:55:44 -0600 Subject: [PATCH 2/8] drivers/java: enable setting allow_caps on java driver Enable setting allow_caps on the java task driver plugin, along with the associated cap_add and cap_drop options in java task configuration. --- drivers/exec/driver.go | 25 +--- drivers/java/driver.go | 82 ++++++++++--- drivers/java/driver_test.go | 116 ++++++++++++++++--- drivers/shared/capabilities/defaults.go | 33 +++++- drivers/shared/capabilities/defaults_test.go | 79 +++++++++++++ 5 files changed, 277 insertions(+), 58 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 4357965083cf..0bd6864c68e1 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -298,29 +298,6 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } -// getCaps computes the complete set of linux capabilities to enable for driver, -// which gets passed along to libcontainer. -func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) { - driverAllowed := capabilities.New(d.config.AllowCaps) - - // determine caps the task wants that are not allowed - taskCaps := capabilities.New(tc.CapAdd) - missing := driverAllowed.Difference(taskCaps) - if !missing.Empty() { - return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) - } - - // if task did not specify allowed caps, use nomad defaults minus task drops - if len(tc.CapAdd) == 0 { - driverAllowed.Remove(tc.CapDrop) - return driverAllowed.Slice(true), nil - } - - // otherwise task did specify allowed caps, enable exactly those - taskAdd := capabilities.New(tc.CapAdd) - return taskAdd.Slice(true), nil -} - // Capabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { @@ -496,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := d.getCaps(&driverConfig) + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) if err != nil { return nil, nil, err } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index c34888f6ea98..a34a56af4f1c 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -73,6 +74,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -88,11 +93,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what + // driverCapabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports - capabilities = &drivers.Capabilities{ + driverCapabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, FSIsolation: drivers.FSIsolationNone, @@ -108,8 +115,8 @@ var ( func init() { if runtime.GOOS == "linux" { - capabilities.FSIsolation = drivers.FSIsolationChroot - capabilities.MountConfigs = drivers.MountConfigSupportAll + driverCapabilities.FSIsolation = drivers.FSIsolationChroot + driverCapabilities.MountConfigs = drivers.MountConfigSupportAll } } @@ -122,6 +129,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -137,18 +148,44 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } // TaskConfig is the driver configuration of a taskConfig within a job type TaskConfig struct { - Class string `codec:"class"` - ClassPath string `codec:"class_path"` - JarPath string `codec:"jar_path"` - JvmOpts []string `codec:"jvm_options"` - Args []string `codec:"args"` // extra arguments to java executable - ModePID string `codec:"pid_mode"` - ModeIPC string `codec:"ipc_mode"` + // Class indicates which class contains the java entry point. + Class string `codec:"class"` + + // ClassPath indicates where class files are found. + ClassPath string `codec:"class_path"` + + // JarPath indicates where a jar file is found. + JarPath string `codec:"jar_path"` + + // JvmOpts are arguments to pass to the JVM + JvmOpts []string `codec:"jvm_options"` + + // Args are extra arguments to java executable + Args []string `codec:"args"` + + // ModePID indicates whether PID namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModePID string `codec:"pid_mode"` + + // ModeIPC indicates whether IPC namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { } func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", - FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot, + FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot, } exec, pluginClient, err := executor.CreateExecutor( @@ -438,6 +485,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + if err != nil { + return nil, nil, err + } + execCmd := &executor.ExecCommand{ Cmd: absPath, Args: args, @@ -453,6 +505,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -491,7 +544,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive } func javaCmdArgs(driverConfig TaskConfig) []string { - args := []string{} + var args []string + // Look for jvm options if len(driverConfig.JvmOpts) != 0 { args = append(args, driverConfig.JvmOpts...) diff --git a/drivers/java/driver_test.go b/drivers/java/driver_test.go index 5c56240375e2..e407e3c19089 100644 --- a/drivers/java/driver_test.go +++ b/drivers/java/driver_test.go @@ -1,6 +1,7 @@ package java import ( + "context" "errors" "fmt" "io" @@ -8,12 +9,10 @@ import ( "os" "path/filepath" "testing" + "time" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "context" - "time" - ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/testlog" @@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) +} + +func TestDriver_TaskConfig_validate(t *testing.T) { + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go index a1a6e3531e0e..30a69b65b491 100644 --- a/drivers/shared/capabilities/defaults.go +++ b/drivers/shared/capabilities/defaults.go @@ -1,6 +1,7 @@ package capabilities import ( + "fmt" "regexp" "github.com/syndtr/gocapability/capability" @@ -13,7 +14,7 @@ const ( ) var ( - extractLiteral = regexp.MustCompile(`("[\w]+)`) + extractLiteral = regexp.MustCompile(`([\w]+)`) ) // NomadDefaults is the set of Linux capabilities that Nomad enables by @@ -112,3 +113,33 @@ func LegacySupported() *Set { "CAP_AUDIT_READ", }) } + +// Calculate the reduced set of linux capabilities to enable for driver, taking +// into account the capabilities allowed by the driver and the capabilities +// explicitly requested / removed by the task configuration. +// +// capAdd if set indicates the minimal set of capabilities that should be enabled. +// capDrop if set indicates capabilities that should be dropped from the driver defaults +// +// If the task requests a capability not allowed by the driver, an error is +// returned. +func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) { + driverAllowed := New(allowCaps) + + // determine caps the task wants that are not allowed + taskCaps := New(capAdd) + missing := driverAllowed.Difference(taskCaps) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // if task did not specify allowed caps, use nomad defaults minus task drops + if len(capAdd) == 0 { + driverAllowed.Remove(capDrop) + return driverAllowed.Slice(true), nil + } + + // otherwise task did specify allowed caps, enable exactly those + taskAdd := New(capAdd) + return taskAdd.Slice(true), nil +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 2a07afa0ea23..2fd9f70d4884 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -1,6 +1,7 @@ package capabilities import ( + "errors" "strings" "testing" @@ -21,3 +22,81 @@ func TestSet_DockerDefaults(t *testing.T) { require.Len(t, result.Slice(false), 14) require.Contains(t, result.String(), "net_raw") } + +func TestCaps_Calculate(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + exp []string + err error + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow all", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + exp: Supported().Slice(true), + err: nil, + }, + { + name: "allow selection", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, + err: nil, + }, + { + name: "add allowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_KILL"}, + err: nil, + }, + { + name: "add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "drop some", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"}, + err: nil, + }, + { + name: "drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop) + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + }) + } +} From c34beb48b1c21479651c68158ac9a19d95ab52d1 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 14:48:01 -0600 Subject: [PATCH 3/8] drivers/docker: reuse capabilities plumbing in docker driver This changeset does not introduce any functional change for the docker driver, but rather cleans up the implementation around computing configured capabilities by re-using code written for the exec/java task drivers. --- drivers/docker/driver.go | 122 +----------- drivers/docker/driver_test.go | 191 ++----------------- drivers/exec/driver.go | 4 +- drivers/java/driver.go | 4 +- drivers/shared/capabilities/defaults.go | 100 ++++++++-- drivers/shared/capabilities/defaults_test.go | 178 ++++++++++++++++- drivers/shared/capabilities/set.go | 27 ++- drivers/shared/capabilities/set_test.go | 52 +++++ 8 files changed, 353 insertions(+), 325 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index c8a2981ec718..7234d34672b8 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "runtime" - "sort" "strconv" "strings" "sync" @@ -23,10 +22,9 @@ import ( plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/docker/docklog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/drivers/shared/eventer" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/drivers/shared/resolvconf" - "github.com/hashicorp/nomad/helper" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -913,8 +911,9 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Privileged = driverConfig.Privileged // set add/drop capabilities - hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig) - if err != nil { + if hostConfig.CapAdd, hostConfig.CapDrop, err = capabilities.Delta( + capabilities.DockerDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ); err != nil { return c, err } @@ -1184,119 +1183,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T }, nil } -// getCaps computes the capabilities to supply to the --add-cap and --drop-cap -// options to the docker driver, which override the default capabilities enabled -// by docker itself. -func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) { - - // capabilities allowable by client docker plugin configuration - allowCaps := expandAllowCaps(d.config.AllowCaps) - - // capabilities the task docker config is asking for based on the default - // capabilities allowable by nomad - desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop) - if err != nil { - return nil, nil, err - } - - // capabilities the task is requesting that are NOT allowed by the docker plugin - if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 { - return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing) - } - - // capabilities that should be dropped relative to the docker default capabilities - dropCaps := capDrops(taskConfig.CapDrop, allowCaps) - - return taskConfig.CapAdd, dropCaps, nil -} - -// capDrops will compute the total dropped capabilities set -// -// {task cap_drop} U ({docker defaults} \ {driver allow caps}) -func capDrops(dropCaps []string, allowCaps []string) []string { - dropSet := make(map[string]struct{}) - - for _, c := range normalizeCaps(dropCaps) { - dropSet[c] = struct{}{} - } - - // if dropCaps includes ALL, no need to iterate every capability - if _, exists := dropSet["ALL"]; exists { - return []string{"ALL"} - } - - dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps())) - allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps)) - - // find the docker default caps not in allowed caps - for dCap := range dockerDefaults { - if _, exists := allowedCaps[dCap]; !exists { - dropSet[dCap] = struct{}{} - } - } - - drops := make([]string, 0, len(dropSet)) - for c := range dropSet { - drops = append(drops, c) - } - sort.Strings(drops) - return drops -} - -// expandAllowCaps returns the normalized set of allowable capabilities set -// for the docker plugin configuration. -func expandAllowCaps(allowCaps []string) []string { - if len(allowCaps) == 0 { - return nil - } - - set := make(map[string]struct{}, len(allowCaps)) - - for _, rawCap := range allowCaps { - capability := strings.ToUpper(rawCap) - if capability == "ALL" { - for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) { - set[defCap] = struct{}{} - } - } else { - set[capability] = struct{}{} - } - } - - result := make([]string, 0, len(set)) - for capability := range set { - result = append(result, capability) - } - sort.Strings(result) - return result -} - -// missingCaps returns the set of elements in desired that are not present in -// allowed. The elements in desired are first upper-cased before comparison. -// The elements in allowed are assumed to be upper-cased. -func missingCaps(allowed, desired []string) []string { - _, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired)) - sort.Strings(missing) - return missing -} - -// normalizeCaps returns a copy of caps with duplicate elements removed and all -// elements upper-cased. -func normalizeCaps(caps []string) []string { - set := make(map[string]struct{}, len(caps)) - for _, c := range caps { - normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_") - set[strings.ToUpper(normal)] = struct{}{} - } - - result := make([]string, 0, len(set)) - for c := range set { - result = append(result, c) - } - sort.Strings(result) - return result -} - func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) { hm, err := m.toDockerHostMount() if err != nil { diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 67c22e88eb31..f51b8b3f07a9 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -19,7 +19,6 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/helper/freeport" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" @@ -1387,44 +1386,44 @@ func TestDockerDriver_Capabilities(t *testing.T) { { Name: "default-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, }, { Name: "default-allowlist-add-forbidden", CapAdd: []string{"net_admin"}, - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "default-allowlist-drop-existing", - CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + CapDrop: []string{"fowner", "mknod", "net_raw"}, }, { Name: "restrictive-allowlist-drop-all", - CapDrop: []string{"ALL"}, - Allowlist: "FOWNER,MKNOD", + CapDrop: []string{"all"}, + Allowlist: "fowner,mknod", }, { Name: "restrictive-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, - Allowlist: "fowner,mknod", + CapDrop: []string{"all"}, + Allowlist: "mknod,fowner", }, { Name: "restrictive-allowlist-add-forbidden", CapAdd: []string{"net_admin", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, Allowlist: "fowner,mknod", - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "permissive-allowlist", - CapAdd: []string{"net_admin", "mknod"}, - Allowlist: "ALL", + CapAdd: []string{"mknod", "net_admin"}, + Allowlist: "all", }, { Name: "permissive-allowlist-add-all", CapAdd: []string{"all"}, - Allowlist: "ALL", + Allowlist: "all", }, } @@ -3064,169 +3063,3 @@ func TestDockerDriver_StopSignal(t *testing.T) { }) } } - -func TestDockerCaps_normalizeCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := normalizeCaps(nil) - require.Len(t, result, 0) - }) - - t.Run("mixed", func(t *testing.T) { - result := normalizeCaps([]string{ - "DAC_OVERRIDE", "sys_chroot", "kill", "KILL", - }) - require.Equal(t, []string{ - "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) -} - -func TestDockerCaps_missingCaps(t *testing.T) { - allowed := []string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - } - - t.Run("none missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "SYS_CHROOT", "chown", "KILL", - }) - require.Equal(t, []string(nil), result) - }) - - t.Run("some missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "chown", "audit_write", "SETPCAP", "dac_override", - }) - require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result) - }) -} - -func TestDockerCaps_expandAllowCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := expandAllowCaps(nil) - require.Empty(t, result) - }) - - t.Run("manual", func(t *testing.T) { - result := expandAllowCaps([]string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - }) - require.Equal(t, []string{ - "CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) - - t.Run("all", func(t *testing.T) { - result := expandAllowCaps([]string{"all"}) - exp := normalizeCaps(executor.SupportedCaps(true)) - sort.Strings(exp) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_capDrops(t *testing.T) { - // docker default caps is always the same, task configured drop_caps and - // plugin config allow_caps may be altered - - // This is the 90% use case, where NET_RAW is dropped, as Nomad's default - // capability allow-list is a subset of the docker default cap list. - t.Run("defaults", func(t *testing.T) { - result := capDrops(nil, nomadDefaultCaps()) - require.Equal(t, []string{"NET_RAW"}, result) - }) - - // Users want to use ICMP (ping). - t.Run("enable net_raw", func(t *testing.T) { - result := capDrops(nil, append(nomadDefaultCaps(), "net_raw")) - require.Empty(t, result) - }) - - // The plugin is reduced in ability. - t.Run("enable minimal", func(t *testing.T) { - allow := []string{"setgid", "setuid", "chown", "kill"} - exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"} - result := capDrops(nil, allow) - require.Equal(t, exp, result) - }) - - // The task drops abilities. - t.Run("task drops", func(t *testing.T) { - drops := []string{"audit_write", "fowner", "kill", "chown"} - exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"} - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) - - // Drop all mixed with others. - t.Run("task drops mix", func(t *testing.T) { - drops := []string{"audit_write", "all", "chown"} - exp := []string{"ALL"} // minimized - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_getCaps(t *testing.T) { - testutil.ExecCompatible(t) // tests require linux - - t.Run("defaults", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("enable net_raw", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "net_raw"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Empty(t, drop) - }) - - t.Run("block sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - _, _, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`) - }) - - t.Run("enable sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "sys_time"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.NoError(t, err) - require.Equal(t, []string{"SYS_TIME"}, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("task drops chown", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, - CapDrop: []string{"chown"}, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop) - }) -} diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 0bd6864c68e1..ef95a374cae5 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -473,7 +473,9 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) if err != nil { return nil, nil, err } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index a34a56af4f1c..d3272cef9e8b 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -485,7 +485,9 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) if err != nil { return nil, nil, err } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go index 30a69b65b491..827f41019d4c 100644 --- a/drivers/shared/capabilities/defaults.go +++ b/drivers/shared/capabilities/defaults.go @@ -39,6 +39,10 @@ func DockerDefaults() *Set { // Supported returns the set of capabilities supported by the operating system. // +// This set will expand over time as new capabilities are introduced to the kernel +// and the capability library is updated (which tends to happen to keep up with +// run-container libraries). +// // Defers to a library generated from // https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h func Supported() *Set { @@ -114,32 +118,92 @@ func LegacySupported() *Set { }) } -// Calculate the reduced set of linux capabilities to enable for driver, taking -// into account the capabilities allowed by the driver and the capabilities -// explicitly requested / removed by the task configuration. +// Calculate the resulting set of linux capabilities to enable for a task, taking +// into account: +// - default capability basis +// - driver allowable capabilities +// - task capability drops +// - task capability adds +// +// Nomad establishes a standard set of enabled capabilities allowed by the task +// driver if allow_caps is not set. This is the same set that the task will be +// enabled with by default if allow_caps does not further reduce permissions, +// in which case the task capabilities will also be reduced accordingly. +// +// The task will drop any capabilities specified in cap_drop, and add back +// capabilities specified in cap_add. The task will not be allowed to add capabilities +// not set in the the allow_caps setting (which by default is the same as the basis). // -// capAdd if set indicates the minimal set of capabilities that should be enabled. -// capDrop if set indicates capabilities that should be dropped from the driver defaults +// cap_add takes precedence over cap_drop, enabling the common pattern of dropping +// all capabilities, then adding back the desired smaller set. e.g. +// cap_drop = ["all"] +// cap_add = ["chown", "kill"] // -// If the task requests a capability not allowed by the driver, an error is -// returned. -func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) { - driverAllowed := New(allowCaps) +// Note that the resulting capability names are upper-cased and prefixed with +// "CAP_", which is the expected input for the exec/java driver implementation. +func Calculate(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, error) { + allow := New(allowCaps) + adds := New(capAdd) // determine caps the task wants that are not allowed - taskCaps := New(capAdd) - missing := driverAllowed.Difference(taskCaps) + missing := allow.Difference(adds) if !missing.Empty() { return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) } - // if task did not specify allowed caps, use nomad defaults minus task drops - if len(capAdd) == 0 { - driverAllowed.Remove(capDrop) - return driverAllowed.Slice(true), nil + // the realized enabled capabilities starts with what is allowed both by driver + // config AND is a member of the basis (i.e. nomad defaults) + result := basis.Intersect(allow) + + // then remove capabilities the task explicitly drops + result.Remove(capDrop) + + // then add back capabilities the task explicitly adds + return result.Union(adds).Slice(true), nil +} + +// Delta calculates the set of capabilities that must be added and dropped relative +// to a basis to achieve a desired result. The use case is that the docker driver +// assumes a default set (DockerDefault), and we must calculate what to pass into +// --cap-add and --cap-drop on container creation given the inputs of the docker +// plugin config for allow_caps, and the docker task configuration for cap_add and +// cap_drop. Note that the user provided cap_add and cap_drop settings are always +// included, even if they are redundant with the basis (maintaining existing +// behavior, working with existing tests). +// +// Note that the resulting capability names are lower-cased and not prefixed with +// "CAP_", which is the existing style used with the docker driver implementation. +func Delta(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, []string, error) { + all := func(caps []string) bool { + for _, c := range caps { + if normalize(c) == "all" { + return true + } + } + return false + } + + // set of caps allowed by driver + allow := New(allowCaps) + + // determine caps the task wants that are not allowed + missing := allow.Difference(New(capAdd)) + if !missing.Empty() { + return nil, nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // add what the task is asking for + add := New(capAdd).Slice(false) + if all(capAdd) { + add = []string{"all"} + } + + // drop what the task removes plus whatever is in the basis that is not + // in the driver allow configuration + drop := New(allowCaps).Difference(basis).Union(New(capDrop)).Slice(false) + if all(capDrop) { + drop = []string{"all"} } - // otherwise task did specify allowed caps, enable exactly those - taskAdd := New(capAdd) - return taskAdd.Slice(true), nil + return add, drop, nil } diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 2fd9f70d4884..408f954ea0c7 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -45,27 +45,59 @@ func TestCaps_Calculate(t *testing.T) { err: nil, }, { - name: "allow all", + name: "allow all no mods", allowCaps: []string{"all"}, capAdd: nil, capDrop: nil, - exp: Supported().Slice(true), + exp: NomadDefaults().Slice(true), err: nil, }, { - name: "allow selection", + name: "allow selection no mods", allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, capAdd: nil, capDrop: nil, + exp: []string{"CAP_CHOWN"}, + err: nil, + }, + { + name: "allow selection and add them", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: []string{"net_raw", "sys_time"}, + capDrop: nil, exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, err: nil, }, { - name: "add allowed", + name: "allow defaults and add redundant", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"chown", "KILL"}, capDrop: nil, - exp: []string{"CAP_CHOWN", "CAP_KILL"}, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow defaults and add all", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + { + name: "allow defaults and drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + { + name: "allow defaults and drop all and add back some", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_FOWNER"}, err: nil, }, { @@ -92,11 +124,145 @@ func TestCaps_Calculate(t *testing.T) { exp: []string{}, err: nil, }, + { + name: "drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "mknod"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_MKNOD"}, + err: nil, + }, } { t.Run(tc.name, func(t *testing.T) { - caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop) + caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) require.Equal(t, tc.err, err) require.Equal(t, tc.exp, caps) }) } } + +func TestCaps_Delta(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + expAdd []string + expDrop []string + err error + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow all no mods", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{}, + err: nil, + }, + { + name: "allow non-default no mods", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{ + "audit_write", "dac_override", "fowner", "fsetid", + "kill", "mknod", "net_bind_service", "setfcap", + "setgid", "setpcap", "setuid", "sys_chroot"}, + err: nil, + }, + { + name: "allow default add from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + expAdd: []string{"chown", "kill"}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow default add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "allow default drop from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + expAdd: []string{}, + expDrop: []string{"chown", "dac_override", "fowner", "kill", "mknod", "net_raw", "sys_chroot"}, + err: nil, + }, + { + name: "allow default drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + expAdd: []string{}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "task drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "add atop allow all", + allowCaps: []string{"all"}, + capAdd: []string{"chown", "fowner"}, + capDrop: nil, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{}, + err: nil, + }, + { + name: "add all atop all", + allowCaps: []string{"all"}, + capAdd: []string{"all"}, + capDrop: nil, + expAdd: []string{"all"}, + expDrop: []string{}, + err: nil, + }, + { + name: "add all atop defaults", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) + require.Equal(t, tc.err, err) + require.Equal(t, tc.expAdd, add) + require.Equal(t, tc.expDrop, drop) + }) + } +} diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go index 046573e2a409..916e3d4d3d93 100644 --- a/drivers/shared/capabilities/set.go +++ b/drivers/shared/capabilities/set.go @@ -14,7 +14,7 @@ var null = nothing{} // operations, taking care of name normalization, and sentinel value expansions. // // Linux capabilities can be expressed in multiple ways when working with docker -// and/or libcontainer, along with Nomad. +// and/or executor, along with Nomad configuration. // // Capability names may be upper or lower case, and may or may not be prefixed // with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all" @@ -61,6 +61,18 @@ func (s *Set) Remove(caps []string) { } } +// Union returns of Set of elements of both s and b. +func (s *Set) Union(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + data[c] = null + } + for c := range b.data { + data[c] = null + } + return &Set{data: data} +} + // Difference returns the Set of elements of b not in s. func (s *Set) Difference(b *Set) *Set { data := make(map[string]nothing) @@ -72,6 +84,17 @@ func (s *Set) Difference(b *Set) *Set { return &Set{data: data} } +// Intersect returns the Set of elements in both s and b. +func (s *Set) Intersect(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + if _, exists := b.data[c]; exists { + data[c] = null + } + } + return &Set{data: data} +} + // Empty return true if no capabilities exist in s. func (s *Set) Empty() bool { return len(s.data) == 0 @@ -84,7 +107,7 @@ func (s *Set) String() string { // Slice returns a sorted slice of capabilities in s. // -// big - indicates whether to uppercase and prefix capabilities with CAP_ +// upper - indicates whether to uppercase and prefix capabilities with CAP_ func (s *Set) Slice(upper bool) []string { caps := make([]string, 0, len(s.data)) for c := range s.data { diff --git a/drivers/shared/capabilities/set_test.go b/drivers/shared/capabilities/set_test.go index 8e072e8b0f28..2134719f248c 100644 --- a/drivers/shared/capabilities/set_test.go +++ b/drivers/shared/capabilities/set_test.go @@ -160,3 +160,55 @@ func TestSet_Difference(t *testing.T) { require.Equal(t, "x, y", result.String()) }) } + +func TestSet_Intersect(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Intersect(b) + require.True(t, result.Empty()) + + result2 := b.Intersect(a) + require.True(t, result2.Empty()) + }) + + t.Run("intersect", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Intersect(b) + require.Equal(t, "b, e, f", result.String()) + + result2 := b.Intersect(a) + require.Equal(t, "b, e, f", result2.String()) + }) +} + +func TestSet_Union(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Union(b) + require.Equal(t, "a, b", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b", result2.String()) + }) + + t.Run("union", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Union(b) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result2.String()) + }) +} From 7245ac3fc511830cf7e31e174aab06a1067dea00 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 17:19:23 -0600 Subject: [PATCH 4/8] docs: update docs for linux capabilities in exec/java/docker drivers Update docs for allow_caps, cap_add, cap_drop in exec/java/docker driver pages. Also update upgrade guide with guidance on new default linux capabilities for exec and java drivers. --- drivers/shared/capabilities/set.go | 1 + website/content/docs/drivers/docker.mdx | 61 +++++++++++-------- website/content/docs/drivers/exec.mdx | 43 +++++++++++++ website/content/docs/drivers/java.mdx | 43 +++++++++++++ .../content/docs/upgrade/upgrade-specific.mdx | 40 +++++++++++- 5 files changed, 158 insertions(+), 30 deletions(-) diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go index 916e3d4d3d93..d6fe527d9d72 100644 --- a/drivers/shared/capabilities/set.go +++ b/drivers/shared/capabilities/set.go @@ -23,6 +23,7 @@ type Set struct { data map[string]nothing } +// New creates a new Set setting caps as the initial elements. func New(caps []string) *Set { m := make(map[string]nothing, len(caps)) for _, c := range caps { diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 68e2bf4f4665..3c202e107a26 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -452,30 +452,26 @@ config { - `cap_add` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-add`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_add = [ - "SYS_TIME", - ] - } - ``` +```hcl +config { + cap_add = ["net_raw", sys_time"] +} +``` - `cap_drop` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-drop`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_drop = [ - "MKNOD", - ] - } - ``` +```hcl +config { + cap_drop = ["mknod"] +} +``` - `cpu_hard_limit` - (Optional) `true` or `false` (default). Use hard CPU limiting instead of soft limiting. By default this is `false` which means @@ -797,10 +793,7 @@ plugin "docker" { } allow_privileged = false - allow_caps = ["CHOWN", "NET_RAW"] - - # allow_caps can also be set to "ALL" - # allow_caps = ["ALL"] + allow_caps = ["chown", "net_raw"] } } ``` @@ -823,13 +816,22 @@ plugin "docker" { from the Docker engine during an image pull within this timeframe, Nomad will timeout the request that initiated the pull command. (Minimum of `1m`) -- `allow_caps` - A list of allowed Linux capabilities. - Defaults to - `CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP, NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE` which is the list of - capabilities allowed by docker by default, as defined here. Allows the - operator to control which capabilities can be obtained by tasks using cap_add - and cap_drop options. Supports the value "ALL" as a shortcut for allowlisting - all capabilities. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is the same list of capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained + by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports + the value `"all"` as a shortcut for allow-listing all capabilities supported by + the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. - `allow_runtimes` - defaults to `["runc", "nvidia"]` - A list of the allowed docker runtimes a task may use. @@ -1136,3 +1138,8 @@ Windows is relatively new and rapidly evolving you may want to consult the [plugin-stanza]: /docs/configuration/plugin [allocation working directory]: /docs/runtime/environment#task-directories 'Task Directories' [`auth_soft_fail=true`]: #auth_soft_fail +[cap_add]: /docs/drivers/docker#cap_add +[cap_drop]: /docs/drivers/docker#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +[allow_caps]: /docs/drivers/docker#allow_caps diff --git a/website/content/docs/drivers/exec.mdx b/website/content/docs/drivers/exec.mdx index 71e491c99f22..df557b7c7db1 100644 --- a/website/content/docs/drivers/exec.mdx +++ b/website/content/docs/drivers/exec.mdx @@ -54,6 +54,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples To run a binary present on the Node: @@ -138,6 +159,23 @@ able to make use of IPC features, like sending unexpected POSIX signals. for file system isolation without `pivot_root`. This is useful for systems where the root is on a ramdisk. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Attributes The `exec` driver will set the following client attributes: @@ -200,3 +238,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/exec#default_pid_mode [default_ipc_mode]: /docs/drivers/exec#default_ipc_mode +[cap_add]: /docs/drivers/exec#cap_add +[cap_drop]: /docs/drivers/exec#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/exec#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index 8124d491cfdb..f7e180a52218 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -61,6 +61,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples A simple config block to run a Java Jar: @@ -138,6 +159,23 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Requirements The `java` driver requires Java to be installed and in your system's `$PATH`. On @@ -208,3 +246,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/java#default_pid_mode [default_ipc_mode]: /docs/drivers/java#default_ipc_mode +[cap_add]: /docs/drivers/java#cap_add +[cap_drop]: /docs/drivers/java#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/java#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 2ff1ade19bc7..e74d16f3ca32 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -54,7 +54,36 @@ these fields. Connect native tasks running in host networking mode will now have `CONSUL_HTTP_ADDR` set automatically. Before this was only the case for bridge networking. If an operator -already explicitly set `CONSUL_HTTP_ADDR` then it will not get overriden. +already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden. + +#### Linux capabilities in exec/java + +Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5, +and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable +the following linux capabilities by default: + +``` +AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE +LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST SYS_ADMIN +SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE +SYS_TIME SYS_TTY_CONFIG WAKE_ALARM +``` + +The capabilities now enabled by default are modeled after Docker default [`linux capabilities`]: + +``` +AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE +NET_RAW SETFCAP SETGID SETPCAP SETUID SYS_CHROOT +``` + +A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec] +and [`java`][allow_caps_java] task drivers can be used to restrict the set of +capabilities allowed for use by tasks. + +Tasks using the `exec` or `java` task drivers can add or remove desired linux +capabilities using the [`cap_add`][cap_add_exec] and [`cap_drop`][cap_drop_exec] +task configuration options. + #### iptables @@ -63,9 +92,9 @@ inserting them as the first rule. This allows better control for user-defined iptables rules but users who append rules currently should verify that their rules are being appended in the correct order. -## Nomad 1.1.0, 1.0.5, 0.12.12 +## Nomad 1.1.0-rc1, 1.0.5, 0.12.12 -Nomad versions 1.1.0, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, +Nomad versions 1.1.0-rc1, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, and `java` task drivers so that the [`CAP_NET_RAW`] linux capability is disabled by default. This is one of the [`linux capabilities`] that Docker itself enables by default, as this capability enables the generation of ICMP packets - used by @@ -1111,3 +1140,8 @@ deleted and then Nomad 0.3.0 can be launched. [`CAP_NET_RAW`]: https://security.stackexchange.com/a/128988 [`linux capabilities`]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities [`allow_caps`]: /docs/drivers/docker#allow_caps +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps_exec]: /docs/drivers/exec#allow_caps +[allow_caps_java]: /docs/drivers/java#allow_caps +[cap_add_exec]: /docs/drivers/exec#cap_add +[cap_drop_exec]: /docs/drivers/exec#cap_drop From 17ec5a5aa8218c0ff529b26076a59db96ea91c55 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 17:45:39 -0600 Subject: [PATCH 5/8] drivers: fixup linux version dependent test cases The error output being checked depends on the linux caps supported by the particular operating system. Fix these test cases to just check that an error did occur. --- drivers/docker/config.go | 30 -------------------- drivers/docker/driver_default.go | 21 -------------- drivers/shared/capabilities/defaults_test.go | 28 +++++++++++++----- 3 files changed, 21 insertions(+), 58 deletions(-) diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 898d9172c870..aff5c5d69690 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -42,36 +42,6 @@ const ( dockerAuthHelperPrefix = "docker-credential-" ) -// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by -// default and is used to compute the set of capabilities to add/drop given -// docker driver configuration. -func nomadDefaultCaps() []string { - return []string{ - "AUDIT_WRITE", - "CHOWN", - "DAC_OVERRIDE", - "FOWNER", - "FSETID", - "KILL", - "MKNOD", - "NET_BIND_SERVICE", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_CHROOT", - } -} - -// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default -// and is used to compute the set of capabilities to add/drop given docker driver -// configuration, as well as Nomad built-in limitations. -// -// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -func dockerDefaultCaps() []string { - return append(nomadDefaultCaps(), "NET_RAW") -} - func PluginLoader(opts map[string]string) (map[string]interface{}, error) { conf := map[string]interface{}{} if v, ok := opts["docker.endpoint"]; ok { diff --git a/drivers/docker/driver_default.go b/drivers/docker/driver_default.go index 45086f7bff0f..b180ae8f92bf 100644 --- a/drivers/docker/driver_default.go +++ b/drivers/docker/driver_default.go @@ -3,30 +3,9 @@ package docker import ( - "github.com/docker/docker/oci/caps" docker "github.com/fsouza/go-dockerclient" ) func getPortBinding(ip string, port string) docker.PortBinding { return docker.PortBinding{HostIP: ip, HostPort: port} } - -func tweakCapabilities(basics, adds, drops []string) ([]string, error) { - // Moby mixes 2 different capabilities formats: prefixed with "CAP_" - // and not. We do the conversion here to have a consistent, - // non-prefixed format on the Nomad side. - for i, cap := range basics { - basics[i] = "CAP_" + cap - } - - effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops, nil, false) - if err != nil { - return effectiveCaps, err - } - - for i, cap := range effectiveCaps { - effectiveCaps[i] = cap[len("CAP_"):] - } - - return effectiveCaps, nil -} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 408f954ea0c7..7fd03513ea8f 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -33,8 +33,9 @@ func TestCaps_Calculate(t *testing.T) { capDrop []string // task config // output - exp []string - err error + exp []string + err error + skip bool // error message is linux version dependent }{ { name: "the default setting", @@ -77,6 +78,7 @@ func TestCaps_Calculate(t *testing.T) { err: nil, }, { + skip: true, name: "allow defaults and add all", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"all"}, @@ -135,8 +137,13 @@ func TestCaps_Calculate(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) - require.Equal(t, tc.err, err) - require.Equal(t, tc.exp, caps) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + } else { + require.Error(t, err) + require.Equal(t, tc.exp, caps) + } }) } } @@ -154,6 +161,7 @@ func TestCaps_Delta(t *testing.T) { expAdd []string expDrop []string err error + skip bool // error message is linux version dependent }{ { name: "the default setting", @@ -249,6 +257,7 @@ func TestCaps_Delta(t *testing.T) { err: nil, }, { + skip: true, name: "add all atop defaults", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"all"}, @@ -260,9 +269,14 @@ func TestCaps_Delta(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) - require.Equal(t, tc.err, err) - require.Equal(t, tc.expAdd, add) - require.Equal(t, tc.expDrop, drop) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.expAdd, add) + require.Equal(t, tc.expDrop, drop) + } else { + require.Error(t, err) + require.Equal(t, tc.expDrop, drop) + } }) } } From 683751ddd63cf0bc1b1842698d5b491b0ac35025 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 18:22:06 -0600 Subject: [PATCH 6/8] deps: update go mod tidy Looks like we no longer need a package. --- .../docker/docker/oci/caps/defaults.go | 21 --- .../docker/docker/oci/caps/utils.go | 169 ------------------ vendor/modules.txt | 1 - 3 files changed, 191 deletions(-) delete mode 100644 vendor/github.com/docker/docker/oci/caps/defaults.go delete mode 100644 vendor/github.com/docker/docker/oci/caps/utils.go diff --git a/vendor/github.com/docker/docker/oci/caps/defaults.go b/vendor/github.com/docker/docker/oci/caps/defaults.go deleted file mode 100644 index 242ee5811d2a..000000000000 --- a/vendor/github.com/docker/docker/oci/caps/defaults.go +++ /dev/null @@ -1,21 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -// DefaultCapabilities returns a Linux kernel default capabilities -func DefaultCapabilities() []string { - return []string{ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE", - } -} diff --git a/vendor/github.com/docker/docker/oci/caps/utils.go b/vendor/github.com/docker/docker/oci/caps/utils.go deleted file mode 100644 index ffd3f6f508ce..000000000000 --- a/vendor/github.com/docker/docker/oci/caps/utils.go +++ /dev/null @@ -1,169 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -import ( - "fmt" - "strings" - - "github.com/docker/docker/errdefs" - "github.com/syndtr/gocapability/capability" -) - -var capabilityList Capabilities - -func init() { - last := capability.CAP_LAST_CAP - // hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap - if last == capability.Cap(63) { - last = capability.CAP_BLOCK_SUSPEND - } - for _, cap := range capability.List() { - if cap > last { - continue - } - capabilityList = append(capabilityList, - &CapabilityMapping{ - Key: "CAP_" + strings.ToUpper(cap.String()), - Value: cap, - }, - ) - } -} - -type ( - // CapabilityMapping maps linux capability name to its value of capability.Cap type - // Capabilities is one of the security systems in Linux Security Module (LSM) - // framework provided by the kernel. - // For more details on capabilities, see http://man7.org/linux/man-pages/man7/capabilities.7.html - CapabilityMapping struct { - Key string `json:"key,omitempty"` - Value capability.Cap `json:"value,omitempty"` - } - // Capabilities contains all CapabilityMapping - Capabilities []*CapabilityMapping -) - -// String returns of CapabilityMapping -func (c *CapabilityMapping) String() string { - return c.Key -} - -// GetCapability returns CapabilityMapping which contains specific key -func GetCapability(key string) *CapabilityMapping { - for _, capp := range capabilityList { - if capp.Key == key { - cpy := *capp - return &cpy - } - } - return nil -} - -// GetAllCapabilities returns all of the capabilities -func GetAllCapabilities() []string { - output := make([]string, len(capabilityList)) - for i, capability := range capabilityList { - output[i] = capability.String() - } - return output -} - -// inSlice tests whether a string is contained in a slice of strings or not. -func inSlice(slice []string, s string) bool { - for _, ss := range slice { - if s == ss { - return true - } - } - return false -} - -const allCapabilities = "ALL" - -// NormalizeLegacyCapabilities normalizes, and validates CapAdd/CapDrop capabilities -// by upper-casing them, and adding a CAP_ prefix (if not yet present). -// -// This function also accepts the "ALL" magic-value, that's used by CapAdd/CapDrop. -func NormalizeLegacyCapabilities(caps []string) ([]string, error) { - var normalized []string - - valids := GetAllCapabilities() - for _, c := range caps { - c = strings.ToUpper(c) - if c == allCapabilities { - normalized = append(normalized, c) - continue - } - if !strings.HasPrefix(c, "CAP_") { - c = "CAP_" + c - } - if !inSlice(valids, c) { - return nil, errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - normalized = append(normalized, c) - } - return normalized, nil -} - -// ValidateCapabilities validates if caps only contains valid capabilities -func ValidateCapabilities(caps []string) error { - valids := GetAllCapabilities() - for _, c := range caps { - if !inSlice(valids, c) { - return errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - } - return nil -} - -// TweakCapabilities tweaks capabilities by adding, dropping, or overriding -// capabilities in the basics capabilities list. -func TweakCapabilities(basics, adds, drops, capabilities []string, privileged bool) ([]string, error) { - switch { - case privileged: - // Privileged containers get all capabilities - return GetAllCapabilities(), nil - case capabilities != nil: - // Use custom set of capabilities - if err := ValidateCapabilities(capabilities); err != nil { - return nil, err - } - return capabilities, nil - case len(adds) == 0 && len(drops) == 0: - // Nothing to tweak; we're done - return basics, nil - } - - capDrop, err := NormalizeLegacyCapabilities(drops) - if err != nil { - return nil, err - } - capAdd, err := NormalizeLegacyCapabilities(adds) - if err != nil { - return nil, err - } - - var caps []string - - switch { - case inSlice(capAdd, allCapabilities): - // Add all capabilities except ones on capDrop - for _, c := range GetAllCapabilities() { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - case inSlice(capDrop, allCapabilities): - // "Drop" all capabilities; use what's in capAdd instead - caps = capAdd - default: - // First drop some capabilities - for _, c := range basics { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - // Then add the list of capabilities from capAdd - caps = append(caps, capAdd...) - } - return caps, nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 37bcd8f6c814..063bfceb28af 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -241,7 +241,6 @@ github.com/docker/docker/api/types/swarm github.com/docker/docker/api/types/swarm/runtime github.com/docker/docker/api/types/versions github.com/docker/docker/errdefs -github.com/docker/docker/oci/caps github.com/docker/docker/pkg/archive github.com/docker/docker/pkg/fileutils github.com/docker/docker/pkg/homedir From 595cef8136e9badd44fdbabebee112101c189e26 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sun, 16 May 2021 10:06:03 -0600 Subject: [PATCH 7/8] drivers/exec: pass capabilities through executor RPC Add capabilities to the LaunchRequest proto so that the capabilities set actually gets plumbed all the way through to task launch. --- drivers/exec/driver.go | 1 + drivers/java/driver.go | 1 + drivers/shared/executor/client.go | 1 + drivers/shared/executor/executor.go | 4 + drivers/shared/executor/executor_linux.go | 1 - drivers/shared/executor/proto/executor.pb.go | 143 ++++++++++--------- drivers/shared/executor/proto/executor.proto | 1 + drivers/shared/executor/server.go | 1 + 8 files changed, 85 insertions(+), 68 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index ef95a374cae5..422b8b42aab9 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -479,6 +479,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err != nil { return nil, nil, err } + d.logger.Debug("task capabilities", "capabilities", caps) execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, diff --git a/drivers/java/driver.go b/drivers/java/driver.go index d3272cef9e8b..f5951ad2b433 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -491,6 +491,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err != nil { return nil, nil, err } + d.logger.Debug("task capabilities", "capabilities", caps) execCmd := &executor.ExecCommand{ Cmd: absPath, diff --git a/drivers/shared/executor/client.go b/drivers/shared/executor/client.go index 67724cc83e19..1779d919a1de 100644 --- a/drivers/shared/executor/client.go +++ b/drivers/shared/executor/client.go @@ -47,6 +47,7 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) { NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation), DefaultPidMode: cmd.ModePID, DefaultIpcMode: cmd.ModeIPC, + Capabilities: cmd.Capabilities, } resp, err := c.client.Launch(ctx, req) if err != nil { diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index 7d738e45f638..10a70f0b82cb 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -92,6 +92,10 @@ type Executor interface { // ExecCommand holds the user command, args, and other isolation related // settings. +// +// Important (!): when adding fields, make sure to update the RPC methods in +// grpcExecutorClient.Launch and grpcExecutorServer.Launch. Number of hours +// spent tracking this down: too many. type ExecCommand struct { // Cmd is the command that the user wants to run. Cmd string diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 1bbb2839cbb6..5c4919e83580 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -534,7 +534,6 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc } func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { - switch command.User { case "root": // when running as root, use the legacy set of system capabilities, so diff --git a/drivers/shared/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go index bf6c980c2904..dcf1a38c7d53 100644 --- a/drivers/shared/executor/proto/executor.pb.go +++ b/drivers/shared/executor/proto/executor.pb.go @@ -45,6 +45,7 @@ type LaunchRequest struct { DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"` CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"` AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"` + Capabilities []string `protobuf:"bytes,19,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -201,6 +202,13 @@ func (m *LaunchRequest) GetAllowCaps() []string { return nil } +func (m *LaunchRequest) GetCapabilities() []string { + if m != nil { + return m.Capabilities + } + return nil +} + type LaunchResponse struct { Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -866,73 +874,74 @@ func init() { } var fileDescriptor_66b85426380683f3 = []byte{ - // 1041 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xed, 0x6f, 0x1b, 0xc5, - 0x13, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0xae, 0x7e, 0xca, 0xef, 0x6a, 0x84, 0x6a, 0x0e, - 0x89, 0x5a, 0x50, 0x2e, 0x51, 0xfa, 0x86, 0x84, 0x44, 0x11, 0x49, 0x41, 0x95, 0xd2, 0x28, 0xba, - 0x14, 0x2a, 0xf1, 0x81, 0x63, 0x7b, 0xb7, 0xb5, 0x57, 0xb1, 0x6f, 0x97, 0xdd, 0x3d, 0x27, 0x48, - 0x48, 0x7c, 0xe2, 0x3f, 0x00, 0xa9, 0x7f, 0x2e, 0xda, 0xb7, 0x8b, 0x9d, 0x96, 0xea, 0x5c, 0xc4, - 0x27, 0xdf, 0x8e, 0x9f, 0x67, 0x66, 0x76, 0x67, 0xe6, 0x19, 0xb8, 0x9b, 0x0b, 0xba, 0x20, 0x42, - 0xee, 0xc9, 0x29, 0x16, 0x24, 0xdf, 0x23, 0x97, 0x24, 0x2b, 0x15, 0x13, 0x7b, 0x5c, 0x30, 0xc5, - 0xaa, 0x63, 0x6c, 0x8e, 0xe8, 0x93, 0x29, 0x96, 0x53, 0x9a, 0x31, 0xc1, 0xe3, 0x82, 0xcd, 0x71, - 0x1e, 0xf3, 0x59, 0x39, 0xa1, 0x85, 0x8c, 0x57, 0x71, 0xc3, 0xdb, 0x13, 0xc6, 0x26, 0x33, 0x62, - 0x9d, 0xbc, 0x2c, 0x5f, 0xed, 0x29, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0xf7, - 0x7c, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0x7a, 0xdd, 0x84, 0xfe, 0x31, 0x2e, 0x8b, 0x6c, 0x9a, - 0x90, 0x5f, 0x4a, 0x22, 0x15, 0x1a, 0x40, 0x23, 0x9b, 0xe7, 0x61, 0x30, 0x0a, 0xc6, 0x9d, 0x44, - 0x7f, 0x22, 0x04, 0x9b, 0x58, 0x4c, 0x64, 0xb8, 0x31, 0x6a, 0x8c, 0x3b, 0x89, 0xf9, 0x46, 0x27, - 0xd0, 0x11, 0x44, 0xb2, 0x52, 0x64, 0x44, 0x86, 0x8d, 0x51, 0x30, 0xee, 0x1e, 0xec, 0xc7, 0xff, - 0x94, 0xb8, 0x8b, 0x6f, 0x43, 0xc6, 0x89, 0xe7, 0x25, 0x57, 0x2e, 0xd0, 0x6d, 0xe8, 0x4a, 0x95, - 0xb3, 0x52, 0xa5, 0x1c, 0xab, 0x69, 0xb8, 0x69, 0xa2, 0x83, 0x35, 0x9d, 0x62, 0x35, 0x75, 0x00, - 0x22, 0x84, 0x05, 0x6c, 0x55, 0x00, 0x22, 0x84, 0x01, 0x0c, 0xa0, 0x41, 0x8a, 0x45, 0xd8, 0x34, - 0x49, 0xea, 0x4f, 0x9d, 0x77, 0x29, 0x89, 0x08, 0x5b, 0x06, 0x6b, 0xbe, 0xd1, 0x2d, 0x68, 0x2b, - 0x2c, 0xcf, 0xd3, 0x9c, 0x8a, 0xb0, 0x6d, 0xec, 0x2d, 0x7d, 0x3e, 0xa2, 0x02, 0xdd, 0x81, 0x1d, - 0x9f, 0x4f, 0x3a, 0xa3, 0x73, 0xaa, 0x64, 0xd8, 0x19, 0x05, 0xe3, 0x76, 0xb2, 0xed, 0xcd, 0xc7, - 0xc6, 0x8a, 0xf6, 0xe1, 0x7f, 0x2f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, - 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x9d, 0xda, 0xbf, 0x0e, 0xcd, 0x3f, 0xe8, 0x08, - 0x9a, 0x73, 0x56, 0x16, 0x4a, 0x86, 0xdd, 0x51, 0x63, 0xdc, 0x3d, 0xb8, 0x5b, 0xf3, 0xa9, 0x9e, - 0x69, 0x52, 0xe2, 0xb8, 0xe8, 0x3b, 0x68, 0xe5, 0x64, 0x41, 0xf5, 0x8b, 0xf7, 0x8c, 0x9b, 0xcf, - 0x6b, 0xba, 0x39, 0x32, 0xac, 0xc4, 0xb3, 0xd1, 0x14, 0x6e, 0x16, 0x44, 0x5d, 0x30, 0x71, 0x9e, - 0x52, 0xc9, 0x66, 0x58, 0x51, 0x56, 0x84, 0x7d, 0x53, 0xc4, 0x2f, 0x6b, 0xba, 0x3c, 0xb1, 0xfc, - 0xa7, 0x9e, 0x7e, 0xc6, 0x49, 0x96, 0x0c, 0x8a, 0x6b, 0x56, 0x14, 0x41, 0xbf, 0x60, 0x29, 0xa7, - 0x0b, 0xa6, 0x52, 0xc1, 0x98, 0x0a, 0xb7, 0xcd, 0x1b, 0x75, 0x0b, 0x76, 0xaa, 0x6d, 0x09, 0x63, - 0x0a, 0x8d, 0x61, 0x90, 0x93, 0x57, 0xb8, 0x9c, 0xa9, 0x94, 0xd3, 0x3c, 0x9d, 0xb3, 0x9c, 0x84, - 0x3b, 0xa6, 0x34, 0xdb, 0xce, 0x7e, 0x4a, 0xf3, 0x67, 0x2c, 0x27, 0xcb, 0x48, 0xca, 0x33, 0x8b, - 0x1c, 0xac, 0x20, 0x9f, 0xf2, 0xcc, 0x20, 0x3f, 0x86, 0x7e, 0xc6, 0x4b, 0x49, 0x94, 0xaf, 0xcd, - 0x4d, 0x03, 0xeb, 0x59, 0xa3, 0xab, 0xca, 0x87, 0x00, 0x78, 0x36, 0x63, 0x17, 0x69, 0x86, 0xb9, - 0x0c, 0x91, 0x69, 0x9c, 0x8e, 0xb1, 0x1c, 0x62, 0x2e, 0xa3, 0x9f, 0x61, 0xdb, 0x4f, 0x86, 0xe4, - 0xac, 0x90, 0x04, 0x9d, 0x40, 0xcb, 0x95, 0xdc, 0x8c, 0x47, 0xf7, 0xe0, 0x7e, 0x5c, 0x6f, 0x56, - 0x63, 0xd7, 0x0e, 0x67, 0x0a, 0x2b, 0x92, 0x78, 0x27, 0x51, 0x1f, 0xba, 0x2f, 0x30, 0x55, 0x6e, - 0xf2, 0xa2, 0x9f, 0xa0, 0x67, 0x8f, 0xff, 0x51, 0xb8, 0x63, 0xd8, 0x39, 0x9b, 0x96, 0x2a, 0x67, - 0x17, 0x85, 0x1f, 0xf6, 0x5d, 0x68, 0x4a, 0x3a, 0x29, 0xf0, 0xcc, 0xcd, 0xbb, 0x3b, 0xa1, 0x8f, - 0xa0, 0x37, 0x11, 0x38, 0x23, 0x29, 0x27, 0x82, 0xb2, 0x3c, 0xdc, 0x18, 0x05, 0xe3, 0x46, 0xd2, - 0x35, 0xb6, 0x53, 0x63, 0x8a, 0x10, 0x0c, 0xae, 0xbc, 0xd9, 0x8c, 0xa3, 0x29, 0xec, 0x7e, 0xcf, - 0x73, 0x1d, 0xb4, 0x9a, 0x71, 0x17, 0x68, 0x45, 0x2f, 0x82, 0x7f, 0xad, 0x17, 0xd1, 0x2d, 0xf8, - 0xff, 0x1b, 0x91, 0x5c, 0x12, 0x03, 0xd8, 0xfe, 0x81, 0x08, 0x49, 0x99, 0xbf, 0x65, 0xf4, 0x19, - 0xec, 0x54, 0x16, 0xf7, 0xb6, 0x21, 0xb4, 0x16, 0xd6, 0xe4, 0x6e, 0xee, 0x8f, 0xd1, 0xa7, 0xd0, - 0xd3, 0xef, 0x56, 0x65, 0x3e, 0x84, 0x36, 0x2d, 0x14, 0x11, 0x0b, 0xf7, 0x48, 0x8d, 0xa4, 0x3a, - 0x47, 0x2f, 0xa0, 0xef, 0xb0, 0xce, 0xed, 0xb7, 0xb0, 0x25, 0xb5, 0x61, 0xcd, 0x2b, 0x3e, 0xc7, - 0xf2, 0xdc, 0x3a, 0xb2, 0xf4, 0xe8, 0x0e, 0xf4, 0xcf, 0x4c, 0x25, 0xde, 0x5e, 0xa8, 0x2d, 0x5f, - 0x28, 0x7d, 0x59, 0x0f, 0x74, 0xd7, 0x3f, 0x87, 0xee, 0x93, 0x4b, 0x92, 0x79, 0xe2, 0x43, 0x68, - 0xe7, 0x04, 0xe7, 0x33, 0x5a, 0x10, 0x97, 0xd4, 0x30, 0xb6, 0x8b, 0x23, 0xf6, 0x8b, 0x23, 0x7e, - 0xee, 0x17, 0x47, 0x52, 0x61, 0xfd, 0x1a, 0xd8, 0x78, 0x73, 0x0d, 0x34, 0xae, 0xd6, 0x40, 0x74, - 0x08, 0x3d, 0x1b, 0xcc, 0xdd, 0x7f, 0x17, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, 0x71, - 0x27, 0xf4, 0x01, 0x74, 0xc8, 0x25, 0x55, 0x69, 0xa6, 0x47, 0x76, 0xc3, 0xdc, 0xa0, 0xad, 0x0d, - 0x87, 0x2c, 0x27, 0xd1, 0x1f, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, 0xd5, - 0x9f, 0xef, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x25, 0x9a, 0x65, - 0xf2, 0xee, 0x6b, 0x1b, 0xdc, 0xc1, 0x5f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x85, 0xa6, - 0x9d, 0x7e, 0xf4, 0xa0, 0xee, 0xd4, 0xad, 0xec, 0xd1, 0xe1, 0xc3, 0x75, 0x69, 0xae, 0x7e, 0x37, - 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x5e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xbf, 0x1e, 0xa9, - 0x0a, 0xfa, 0x3b, 0xb4, 0xfd, 0x38, 0xa3, 0x47, 0x75, 0x7d, 0x5c, 0x93, 0x93, 0xe1, 0x17, 0xeb, - 0x13, 0xab, 0x04, 0xfe, 0x0c, 0x60, 0xe7, 0xda, 0x48, 0xa3, 0xaf, 0xea, 0xfa, 0x7b, 0xbb, 0xea, - 0x0c, 0x1f, 0xbf, 0x37, 0xbf, 0x4a, 0xeb, 0x37, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, 0x2a, - 0x3f, 0xc3, 0x47, 0x6b, 0xf3, 0xaa, 0xe8, 0x97, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, 0xb2, - 0x76, 0x0d, 0x1f, 0xac, 0xc9, 0xf2, 0x71, 0xf7, 0x03, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, 0xff, - 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x6b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, 0xd2, - 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x0d, 0xf4, 0x3a, 0x80, 0xbe, 0x36, 0x9d, 0x29, 0x41, - 0xf0, 0x9c, 0x16, 0x13, 0xf4, 0xb8, 0xa6, 0x78, 0x6b, 0x96, 0x15, 0x70, 0xc7, 0xf4, 0xa9, 0x7c, - 0xfd, 0xfe, 0x0e, 0x7c, 0x5a, 0xe3, 0x60, 0x3f, 0xf8, 0xa6, 0xf5, 0xe3, 0x96, 0xd5, 0xac, 0xa6, - 0xf9, 0xb9, 0xf7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x41, 0x13, 0xe3, 0x8e, 0x50, 0x0c, 0x00, - 0x00, + // 1058 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x6d, 0x6f, 0x1b, 0x45, + 0x10, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0x2e, 0xa8, 0x5c, 0x8d, 0x50, 0xcd, 0x21, 0x51, + 0x0b, 0xca, 0x25, 0x4a, 0xdf, 0x90, 0x90, 0x28, 0x22, 0x2d, 0xa8, 0x52, 0x1a, 0x45, 0x97, 0x42, + 0x25, 0x3e, 0x70, 0x6c, 0xee, 0xb6, 0xf6, 0x2a, 0xf6, 0xed, 0xb2, 0xbb, 0xe7, 0x04, 0x09, 0x89, + 0x4f, 0xfc, 0x03, 0x90, 0xf8, 0x31, 0xfc, 0x38, 0xb4, 0x6f, 0x17, 0x3b, 0x2d, 0xd5, 0xb9, 0x88, + 0x4f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x27, 0x17, 0x74, 0x41, 0x84, + 0xdc, 0x95, 0x53, 0x2c, 0x48, 0xbe, 0x4b, 0x2e, 0x48, 0x56, 0x2a, 0x26, 0x76, 0xb9, 0x60, 0x8a, + 0x55, 0xc7, 0xd8, 0x1c, 0xd1, 0xc7, 0x53, 0x2c, 0xa7, 0x34, 0x63, 0x82, 0xc7, 0x05, 0x9b, 0xe3, + 0x3c, 0xe6, 0xb3, 0x72, 0x42, 0x0b, 0x19, 0xaf, 0xe2, 0x86, 0xb7, 0x26, 0x8c, 0x4d, 0x66, 0xc4, + 0x3a, 0x39, 0x2d, 0x5f, 0xee, 0x2a, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0x77, + 0x7d, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0xfa, 0xbb, 0x09, 0xfd, 0x43, 0x5c, 0x16, 0xd9, 0x34, + 0x21, 0x3f, 0x97, 0x44, 0x2a, 0x34, 0x80, 0x46, 0x36, 0xcf, 0xc3, 0x60, 0x14, 0x8c, 0x3b, 0x89, + 0xfe, 0x44, 0x08, 0x36, 0xb1, 0x98, 0xc8, 0x70, 0x63, 0xd4, 0x18, 0x77, 0x12, 0xf3, 0x8d, 0x8e, + 0xa0, 0x23, 0x88, 0x64, 0xa5, 0xc8, 0x88, 0x0c, 0x1b, 0xa3, 0x60, 0xdc, 0xdd, 0xdf, 0x8b, 0xff, + 0x2d, 0x71, 0x17, 0xdf, 0x86, 0x8c, 0x13, 0xcf, 0x4b, 0x2e, 0x5d, 0xa0, 0x5b, 0xd0, 0x95, 0x2a, + 0x67, 0xa5, 0x4a, 0x39, 0x56, 0xd3, 0x70, 0xd3, 0x44, 0x07, 0x6b, 0x3a, 0xc6, 0x6a, 0xea, 0x00, + 0x44, 0x08, 0x0b, 0xd8, 0xaa, 0x00, 0x44, 0x08, 0x03, 0x18, 0x40, 0x83, 0x14, 0x8b, 0xb0, 0x69, + 0x92, 0xd4, 0x9f, 0x3a, 0xef, 0x52, 0x12, 0x11, 0xb6, 0x0c, 0xd6, 0x7c, 0xa3, 0x9b, 0xd0, 0x56, + 0x58, 0x9e, 0xa5, 0x39, 0x15, 0x61, 0xdb, 0xd8, 0x5b, 0xfa, 0xfc, 0x98, 0x0a, 0x74, 0x1b, 0x76, + 0x7c, 0x3e, 0xe9, 0x8c, 0xce, 0xa9, 0x92, 0x61, 0x67, 0x14, 0x8c, 0xdb, 0xc9, 0xb6, 0x37, 0x1f, + 0x1a, 0x2b, 0xda, 0x83, 0x77, 0x4f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, + 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x1d, 0xdb, 0xbf, 0x0e, 0xcc, 0x3f, 0xe8, 0x31, + 0x34, 0xe7, 0xac, 0x2c, 0x94, 0x0c, 0xbb, 0xa3, 0xc6, 0xb8, 0xbb, 0x7f, 0xa7, 0xe6, 0x53, 0x3d, + 0xd3, 0xa4, 0xc4, 0x71, 0xd1, 0xb7, 0xd0, 0xca, 0xc9, 0x82, 0xea, 0x17, 0xef, 0x19, 0x37, 0x9f, + 0xd5, 0x74, 0xf3, 0xd8, 0xb0, 0x12, 0xcf, 0x46, 0x53, 0xb8, 0x5e, 0x10, 0x75, 0xce, 0xc4, 0x59, + 0x4a, 0x25, 0x9b, 0x61, 0x45, 0x59, 0x11, 0xf6, 0x4d, 0x11, 0xbf, 0xa8, 0xe9, 0xf2, 0xc8, 0xf2, + 0x9f, 0x7a, 0xfa, 0x09, 0x27, 0x59, 0x32, 0x28, 0xae, 0x58, 0x51, 0x04, 0xfd, 0x82, 0xa5, 0x9c, + 0x2e, 0x98, 0x4a, 0x05, 0x63, 0x2a, 0xdc, 0x36, 0x6f, 0xd4, 0x2d, 0xd8, 0xb1, 0xb6, 0x25, 0x8c, + 0x29, 0x34, 0x86, 0x41, 0x4e, 0x5e, 0xe2, 0x72, 0xa6, 0x52, 0x4e, 0xf3, 0x74, 0xce, 0x72, 0x12, + 0xee, 0x98, 0xd2, 0x6c, 0x3b, 0xfb, 0x31, 0xcd, 0x9f, 0xb1, 0x9c, 0x2c, 0x23, 0x29, 0xcf, 0x2c, + 0x72, 0xb0, 0x82, 0x7c, 0xca, 0x33, 0x83, 0xfc, 0x08, 0xfa, 0x19, 0x2f, 0x25, 0x51, 0xbe, 0x36, + 0xd7, 0x0d, 0xac, 0x67, 0x8d, 0xae, 0x2a, 0x1f, 0x00, 0xe0, 0xd9, 0x8c, 0x9d, 0xa7, 0x19, 0xe6, + 0x32, 0x44, 0xa6, 0x71, 0x3a, 0xc6, 0x72, 0x80, 0xb9, 0x44, 0x11, 0xf4, 0x32, 0xcc, 0xf1, 0x29, + 0x9d, 0x51, 0x45, 0x89, 0x0c, 0xdf, 0x31, 0x80, 0x15, 0x5b, 0xf4, 0x13, 0x6c, 0xfb, 0xe9, 0x91, + 0x9c, 0x15, 0x92, 0xa0, 0x23, 0x68, 0xb9, 0xb6, 0x30, 0x23, 0xd4, 0xdd, 0xbf, 0x17, 0xd7, 0x9b, + 0xe7, 0xd8, 0xb5, 0xcc, 0x89, 0xc2, 0x8a, 0x24, 0xde, 0x49, 0xd4, 0x87, 0xee, 0x0b, 0x4c, 0x95, + 0x9b, 0xce, 0xe8, 0x47, 0xe8, 0xd9, 0xe3, 0xff, 0x14, 0xee, 0x10, 0x76, 0x4e, 0xa6, 0xa5, 0xca, + 0xd9, 0x79, 0xe1, 0x05, 0xe1, 0x06, 0x34, 0x25, 0x9d, 0x14, 0x78, 0xe6, 0x34, 0xc1, 0x9d, 0xd0, + 0x87, 0xd0, 0x9b, 0x08, 0x9c, 0x91, 0x94, 0x13, 0x41, 0x59, 0x1e, 0x6e, 0x8c, 0x82, 0x71, 0x23, + 0xe9, 0x1a, 0xdb, 0xb1, 0x31, 0x45, 0x08, 0x06, 0x97, 0xde, 0x6c, 0xc6, 0xd1, 0x14, 0x6e, 0x7c, + 0xc7, 0x73, 0x1d, 0xb4, 0xd2, 0x01, 0x17, 0x68, 0x45, 0x53, 0x82, 0xff, 0xac, 0x29, 0xd1, 0x4d, + 0x78, 0xef, 0x95, 0x48, 0x2e, 0x89, 0x01, 0x6c, 0x7f, 0x4f, 0x84, 0xa4, 0xcc, 0xdf, 0x32, 0xfa, + 0x14, 0x76, 0x2a, 0x8b, 0x7b, 0xdb, 0x10, 0x5a, 0x0b, 0x6b, 0x72, 0x37, 0xf7, 0xc7, 0xe8, 0x13, + 0xe8, 0xe9, 0x77, 0xab, 0x32, 0x1f, 0x42, 0x9b, 0x16, 0x8a, 0x88, 0x85, 0x7b, 0xa4, 0x46, 0x52, + 0x9d, 0xa3, 0x17, 0xd0, 0x77, 0x58, 0xe7, 0xf6, 0x1b, 0xd8, 0x92, 0xda, 0xb0, 0xe6, 0x15, 0x9f, + 0x63, 0x79, 0x66, 0x1d, 0x59, 0x7a, 0x74, 0x1b, 0xfa, 0x27, 0xa6, 0x12, 0xaf, 0x2f, 0xd4, 0x96, + 0x2f, 0x94, 0xbe, 0xac, 0x07, 0xba, 0xeb, 0x9f, 0x41, 0xf7, 0xc9, 0x05, 0xc9, 0x3c, 0xf1, 0x01, + 0xb4, 0x73, 0x82, 0xf3, 0x19, 0x2d, 0x88, 0x4b, 0x6a, 0x18, 0xdb, 0xe5, 0x12, 0xfb, 0xe5, 0x12, + 0x3f, 0xf7, 0xcb, 0x25, 0xa9, 0xb0, 0x7e, 0x55, 0x6c, 0xbc, 0xba, 0x2a, 0x1a, 0x97, 0xab, 0x22, + 0x3a, 0x80, 0x9e, 0x0d, 0xe6, 0xee, 0x7f, 0x03, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, + 0x71, 0x27, 0xf4, 0x3e, 0x74, 0xc8, 0x05, 0x55, 0x69, 0xa6, 0xc7, 0x7a, 0xc3, 0xdc, 0xa0, 0xad, + 0x0d, 0x07, 0x2c, 0x27, 0xd1, 0xef, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, + 0xd5, 0x9f, 0x6f, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x6d, 0x9a, + 0x85, 0xf3, 0xe6, 0x6b, 0x1b, 0xdc, 0xfe, 0x9f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x81, + 0xa6, 0x9d, 0x7e, 0x74, 0xbf, 0xee, 0xd4, 0xad, 0xec, 0xda, 0xe1, 0x83, 0x75, 0x69, 0xae, 0x7e, + 0xd7, 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x6e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xb7, 0x1e, + 0xa9, 0x0a, 0xfa, 0x1b, 0xb4, 0xfd, 0x38, 0xa3, 0x87, 0x75, 0x7d, 0x5c, 0x91, 0x93, 0xe1, 0xe7, + 0xeb, 0x13, 0xab, 0x04, 0xfe, 0x08, 0x60, 0xe7, 0xca, 0x48, 0xa3, 0x2f, 0xeb, 0xfa, 0x7b, 0xbd, + 0xea, 0x0c, 0x1f, 0xbd, 0x35, 0xbf, 0x4a, 0xeb, 0x57, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, + 0x2a, 0x3f, 0xc3, 0x87, 0x6b, 0xf3, 0xaa, 0xe8, 0x17, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, + 0xb2, 0x76, 0x0d, 0xef, 0xaf, 0xc9, 0xf2, 0x71, 0xf7, 0x02, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, + 0xff, 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x2b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, + 0xd2, 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x35, 0xf4, 0x57, 0x00, 0x7d, 0x6d, 0x3a, 0x51, + 0x82, 0xe0, 0x39, 0x2d, 0x26, 0xe8, 0x51, 0x4d, 0xf1, 0xd6, 0x2c, 0x2b, 0xe0, 0x8e, 0xe9, 0x53, + 0xf9, 0xea, 0xed, 0x1d, 0xf8, 0xb4, 0xc6, 0xc1, 0x5e, 0xf0, 0x75, 0xeb, 0x87, 0x2d, 0xab, 0x59, + 0x4d, 0xf3, 0x73, 0xf7, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2e, 0x63, 0xc4, 0xd3, 0x74, 0x0c, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/drivers/shared/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto index 9ac241209fcf..aa349d265fac 100644 --- a/drivers/shared/executor/proto/executor.proto +++ b/drivers/shared/executor/proto/executor.proto @@ -46,6 +46,7 @@ message LaunchRequest { string default_ipc_mode = 16; string cpuset_cgroup = 17; repeated string allow_caps = 18; + repeated string capabilities = 19; } message LaunchResponse { diff --git a/drivers/shared/executor/server.go b/drivers/shared/executor/server.go index 7ef91f4d341f..9583bbb7b96d 100644 --- a/drivers/shared/executor/server.go +++ b/drivers/shared/executor/server.go @@ -37,6 +37,7 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation), ModePID: req.DefaultPidMode, ModeIPC: req.DefaultIpcMode, + Capabilities: req.Capabilities, }) if err != nil { From 845a3d3cdc4a9fc97e4cde65dc5c9529971753f2 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Mon, 17 May 2021 12:52:52 -0600 Subject: [PATCH 8/8] docs: minor wording tweaks + cl --- CHANGELOG.md | 1 + website/content/docs/drivers/docker.mdx | 2 +- website/content/docs/drivers/exec.mdx | 2 +- website/content/docs/drivers/java.mdx | 2 +- website/content/docs/upgrade/upgrade-specific.mdx | 9 +++++---- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c944cd749d70..a2dfd29e946c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ FEATURES: __BACKWARDS INCOMPATIBILITIES:__ * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)] + * drivers/exec+java: Reduce set of linux capabilities enabled by default [[GH-10600](https://github.com/hashicorp/nomad/pull/10600)] * licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)] SECURITY: diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 3c202e107a26..16e248be7b9c 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -824,7 +824,7 @@ plugin "docker" { ``` which is the same list of capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/drivers/exec.mdx b/website/content/docs/drivers/exec.mdx index df557b7c7db1..065693233d24 100644 --- a/website/content/docs/drivers/exec.mdx +++ b/website/content/docs/drivers/exec.mdx @@ -167,7 +167,7 @@ able to make use of IPC features, like sending unexpected POSIX signals. ``` which is modeled after the capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index f7e180a52218..ad0957dbf1b0 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -167,7 +167,7 @@ able to make use of IPC features, like sending unexpected POSIX signals. ``` which is modeled after the capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index e74d16f3ca32..0a5c1ac611df 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -60,20 +60,21 @@ already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden. Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5, and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable -the following linux capabilities by default: +the following linux capabilities by default. ``` AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE -LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST SYS_ADMIN +LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST NET_RAW SYS_ADMIN SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE SYS_TIME SYS_TTY_CONFIG WAKE_ALARM ``` -The capabilities now enabled by default are modeled after Docker default [`linux capabilities`]: +The capabilities now enabled by default are modeled after Docker default +[`linux capabilities`] (excluding `NET_RAW`). ``` AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE -NET_RAW SETFCAP SETGID SETPCAP SETUID SYS_CHROOT +SETFCAP SETGID SETPCAP SETUID SYS_CHROOT ``` A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec]