From c53a9486e804dba38be566ab6a071171d8cef590 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 10:55:44 -0600 Subject: [PATCH] drivers/java: enable setting allow_caps on java driver Enable setting allow_caps on the java task driver plugin, along with the associated cap_add and cap_drop options in java task configuration. --- drivers/exec/driver.go | 25 +--- drivers/java/driver.go | 82 ++++++++++--- drivers/java/driver_test.go | 116 ++++++++++++++++--- drivers/shared/capabilities/defaults.go | 33 +++++- drivers/shared/capabilities/defaults_test.go | 79 +++++++++++++ 5 files changed, 277 insertions(+), 58 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 4357965083cf..0bd6864c68e1 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -298,29 +298,6 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } -// getCaps computes the complete set of linux capabilities to enable for driver, -// which gets passed along to libcontainer. -func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) { - driverAllowed := capabilities.New(d.config.AllowCaps) - - // determine caps the task wants that are not allowed - taskCaps := capabilities.New(tc.CapAdd) - missing := driverAllowed.Difference(taskCaps) - if !missing.Empty() { - return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) - } - - // if task did not specify allowed caps, use nomad defaults minus task drops - if len(tc.CapAdd) == 0 { - driverAllowed.Remove(tc.CapDrop) - return driverAllowed.Slice(true), nil - } - - // otherwise task did specify allowed caps, enable exactly those - taskAdd := capabilities.New(tc.CapAdd) - return taskAdd.Slice(true), nil -} - // Capabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { @@ -496,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := d.getCaps(&driverConfig) + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) if err != nil { return nil, nil, err } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index c34888f6ea98..a34a56af4f1c 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -73,6 +74,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -88,11 +93,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what + // driverCapabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports - capabilities = &drivers.Capabilities{ + driverCapabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, FSIsolation: drivers.FSIsolationNone, @@ -108,8 +115,8 @@ var ( func init() { if runtime.GOOS == "linux" { - capabilities.FSIsolation = drivers.FSIsolationChroot - capabilities.MountConfigs = drivers.MountConfigSupportAll + driverCapabilities.FSIsolation = drivers.FSIsolationChroot + driverCapabilities.MountConfigs = drivers.MountConfigSupportAll } } @@ -122,6 +129,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -137,18 +148,44 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } // TaskConfig is the driver configuration of a taskConfig within a job type TaskConfig struct { - Class string `codec:"class"` - ClassPath string `codec:"class_path"` - JarPath string `codec:"jar_path"` - JvmOpts []string `codec:"jvm_options"` - Args []string `codec:"args"` // extra arguments to java executable - ModePID string `codec:"pid_mode"` - ModeIPC string `codec:"ipc_mode"` + // Class indicates which class contains the java entry point. + Class string `codec:"class"` + + // ClassPath indicates where class files are found. + ClassPath string `codec:"class_path"` + + // JarPath indicates where a jar file is found. + JarPath string `codec:"jar_path"` + + // JvmOpts are arguments to pass to the JVM + JvmOpts []string `codec:"jvm_options"` + + // Args are extra arguments to java executable + Args []string `codec:"args"` + + // ModePID indicates whether PID namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModePID string `codec:"pid_mode"` + + // ModeIPC indicates whether IPC namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { } func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", - FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot, + FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot, } exec, pluginClient, err := executor.CreateExecutor( @@ -438,6 +485,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + if err != nil { + return nil, nil, err + } + execCmd := &executor.ExecCommand{ Cmd: absPath, Args: args, @@ -453,6 +505,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -491,7 +544,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive } func javaCmdArgs(driverConfig TaskConfig) []string { - args := []string{} + var args []string + // Look for jvm options if len(driverConfig.JvmOpts) != 0 { args = append(args, driverConfig.JvmOpts...) diff --git a/drivers/java/driver_test.go b/drivers/java/driver_test.go index 5c56240375e2..e407e3c19089 100644 --- a/drivers/java/driver_test.go +++ b/drivers/java/driver_test.go @@ -1,6 +1,7 @@ package java import ( + "context" "errors" "fmt" "io" @@ -8,12 +9,10 @@ import ( "os" "path/filepath" "testing" + "time" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "context" - "time" - ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/testlog" @@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) +} + +func TestDriver_TaskConfig_validate(t *testing.T) { + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go index a1a6e3531e0e..30a69b65b491 100644 --- a/drivers/shared/capabilities/defaults.go +++ b/drivers/shared/capabilities/defaults.go @@ -1,6 +1,7 @@ package capabilities import ( + "fmt" "regexp" "github.com/syndtr/gocapability/capability" @@ -13,7 +14,7 @@ const ( ) var ( - extractLiteral = regexp.MustCompile(`("[\w]+)`) + extractLiteral = regexp.MustCompile(`([\w]+)`) ) // NomadDefaults is the set of Linux capabilities that Nomad enables by @@ -112,3 +113,33 @@ func LegacySupported() *Set { "CAP_AUDIT_READ", }) } + +// Calculate the reduced set of linux capabilities to enable for driver, taking +// into account the capabilities allowed by the driver and the capabilities +// explicitly requested / removed by the task configuration. +// +// capAdd if set indicates the minimal set of capabilities that should be enabled. +// capDrop if set indicates capabilities that should be dropped from the driver defaults +// +// If the task requests a capability not allowed by the driver, an error is +// returned. +func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) { + driverAllowed := New(allowCaps) + + // determine caps the task wants that are not allowed + taskCaps := New(capAdd) + missing := driverAllowed.Difference(taskCaps) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // if task did not specify allowed caps, use nomad defaults minus task drops + if len(capAdd) == 0 { + driverAllowed.Remove(capDrop) + return driverAllowed.Slice(true), nil + } + + // otherwise task did specify allowed caps, enable exactly those + taskAdd := New(capAdd) + return taskAdd.Slice(true), nil +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 2a07afa0ea23..2fd9f70d4884 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -1,6 +1,7 @@ package capabilities import ( + "errors" "strings" "testing" @@ -21,3 +22,81 @@ func TestSet_DockerDefaults(t *testing.T) { require.Len(t, result.Slice(false), 14) require.Contains(t, result.String(), "net_raw") } + +func TestCaps_Calculate(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + exp []string + err error + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow all", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + exp: Supported().Slice(true), + err: nil, + }, + { + name: "allow selection", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, + err: nil, + }, + { + name: "add allowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_KILL"}, + err: nil, + }, + { + name: "add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "drop some", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"}, + err: nil, + }, + { + name: "drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop) + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + }) + } +}