Skip to content

Commit

Permalink
drivers/java: enable setting allow_caps on java driver
Browse files Browse the repository at this point in the history 
Enable setting allow_caps on the java task driver plugin, along
with the associated cap_add and cap_drop options in java task
configuration.
  • Loading branch information
@shoenig
shoenig committed May 15, 2021
1 parent d817d12 commit c53a948
Show file tree
Hide file tree
Showing 5 changed files with 277 additions and 58 deletions.
25 changes: 1 addition & 24 deletions drivers/exec/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,29 +298,6 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
return taskConfigSpec, nil
}

// getCaps computes the complete set of linux capabilities to enable for driver,
// which gets passed along to libcontainer.
func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) {
driverAllowed := capabilities.New(d.config.AllowCaps)

// determine caps the task wants that are not allowed
taskCaps := capabilities.New(tc.CapAdd)
missing := driverAllowed.Difference(taskCaps)
if !missing.Empty() {
return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing)
}

// if task did not specify allowed caps, use nomad defaults minus task drops
if len(tc.CapAdd) == 0 {
driverAllowed.Remove(tc.CapDrop)
return driverAllowed.Slice(true), nil
}

// otherwise task did specify allowed caps, enable exactly those
taskAdd := capabilities.New(tc.CapAdd)
return taskAdd.Slice(true), nil
}

// Capabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
Expand Down Expand Up @@ -496,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}

caps, err := d.getCaps(&driverConfig)
caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop)
if err != nil {
return nil, nil, err
}
Expand Down
82 changes: 68 additions & 14 deletions drivers/java/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

"github.com/hashicorp/nomad/client/lib/cgutil"
"github.com/hashicorp/nomad/drivers/shared/capabilities"

"github.com/hashicorp/consul-template/signals"
hclog "github.com/hashicorp/go-hclog"
Expand Down Expand Up @@ -73,6 +74,10 @@ var (
hclspec.NewAttr("default_ipc_mode", "string", false),
hclspec.NewLiteral(`"private"`),
),
"allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
),
})

// taskConfigSpec is the hcl specification for the driver config section of
Expand All @@ -88,11 +93,13 @@ var (
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
})

// capabilities is returned by the Capabilities RPC and indicates what
// driverCapabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
capabilities = &drivers.Capabilities{
driverCapabilities = &drivers.Capabilities{
SendSignals: false,
Exec: false,
FSIsolation: drivers.FSIsolationNone,
Expand All @@ -108,8 +115,8 @@ var (

func init() {
if runtime.GOOS == "linux" {
capabilities.FSIsolation = drivers.FSIsolationChroot
capabilities.MountConfigs = drivers.MountConfigSupportAll
driverCapabilities.FSIsolation = drivers.FSIsolationChroot
driverCapabilities.MountConfigs = drivers.MountConfigSupportAll
}
}

Expand All @@ -122,6 +129,10 @@ type Config struct {
// DefaultModeIPC is the default IPC isolation set for all tasks using
// exec-based task drivers.
DefaultModeIPC string `codec:"default_ipc_mode"`

// AllowCaps configures which Linux Capabilities are enabled for tasks
// running on this node.
AllowCaps []string `codec:"allow_caps"`
}

func (c *Config) validate() error {
Expand All @@ -137,18 +148,44 @@ func (c *Config) validate() error {
return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
}

badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps))
if !badCaps.Empty() {
return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps)
}

return nil
}

// TaskConfig is the driver configuration of a taskConfig within a job
type TaskConfig struct {
Class string `codec:"class"`
ClassPath string `codec:"class_path"`
JarPath string `codec:"jar_path"`
JvmOpts []string `codec:"jvm_options"`
Args []string `codec:"args"` // extra arguments to java executable
ModePID string `codec:"pid_mode"`
ModeIPC string `codec:"ipc_mode"`
// Class indicates which class contains the java entry point.
Class string `codec:"class"`

// ClassPath indicates where class files are found.
ClassPath string `codec:"class_path"`

// JarPath indicates where a jar file is found.
JarPath string `codec:"jar_path"`

// JvmOpts are arguments to pass to the JVM
JvmOpts []string `codec:"jvm_options"`

// Args are extra arguments to java executable
Args []string `codec:"args"`

// ModePID indicates whether PID namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModePID string `codec:"pid_mode"`

// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"`

// CapAdd is a set of linux capabilities to enable.
CapAdd []string `codec:"cap_add"`

// CapDrop is a set of linux capabilities to disable.
CapDrop []string `codec:"cap_drop"`
}

func (tc *TaskConfig) validate() error {
Expand All @@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error {
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}

supported := capabilities.Supported()
badAdds := supported.Difference(capabilities.New(tc.CapAdd))
if !badAdds.Empty() {
return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds)
}
badDrops := supported.Difference(capabilities.New(tc.CapDrop))
if !badDrops.Empty() {
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
}

return nil
}

Expand Down Expand Up @@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
}

func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
return capabilities, nil
return driverCapabilities, nil
}

func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
Expand Down Expand Up @@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
executorConfig := &executor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: "debug",
FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot,
FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot,
}

exec, pluginClient, err := executor.CreateExecutor(
Expand All @@ -438,6 +485,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}

caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop)
if err != nil {
return nil, nil, err
}

execCmd := &executor.ExecCommand{
Cmd: absPath,
Args: args,
Expand All @@ -453,6 +505,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
NetworkIsolation: cfg.NetworkIsolation,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
Capabilities: caps,
}

ps, err := exec.Launch(execCmd)
Expand Down Expand Up @@ -491,7 +544,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
}

func javaCmdArgs(driverConfig TaskConfig) []string {
args := []string{}
var args []string

// Look for jvm options
if len(driverConfig.JvmOpts) != 0 {
args = append(args, driverConfig.JvmOpts...)
Expand Down
116 changes: 97 additions & 19 deletions drivers/java/driver_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
package java

import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"

dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"

"context"
"time"

ctestutil "github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
"github.com/hashicorp/nomad/helper/testlog"
Expand Down Expand Up @@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) {
}

func TestDriver_Config_validate(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "private", ipcMode: "private", exp: nil},
{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: tc.pidMode,
DefaultModeIPC: tc.ipcMode,
}).validate())
}
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "private", ipcMode: "private", exp: nil},
{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: tc.pidMode,
DefaultModeIPC: tc.ipcMode,
}).validate())
}
})

t.Run("allow_caps", func(t *testing.T) {
for _, tc := range []struct {
ac []string
exp error
}{
{ac: []string{}, exp: nil},
{ac: []string{"all"}, exp: nil},
{ac: []string{"chown", "sys_time"}, exp: nil},
{ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil},
{ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: "private",
DefaultModeIPC: "private",
AllowCaps: tc.ac,
}).validate())
}
})
}

func TestDriver_TaskConfig_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "host", ipcMode: "", exp: nil},
{pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)},

{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "", ipcMode: "host", exp: nil},
{pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&TaskConfig{
ModePID: tc.pidMode,
ModeIPC: tc.ipcMode,
}).validate())
}
})

t.Run("cap_add", func(t *testing.T) {
for _, tc := range []struct {
adds []string
exp error
}{
{adds: nil, exp: nil},
{adds: []string{"chown"}, exp: nil},
{adds: []string{"CAP_CHOWN"}, exp: nil},
{adds: []string{"chown", "sys_time"}, exp: nil},
{adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapAdd: tc.adds,
}).validate())
}
})

t.Run("cap_drop", func(t *testing.T) {
for _, tc := range []struct {
drops []string
exp error
}{
{drops: nil, exp: nil},
{drops: []string{"chown"}, exp: nil},
{drops: []string{"CAP_CHOWN"}, exp: nil},
{drops: []string{"chown", "sys_time"}, exp: nil},
{drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapDrop: tc.drops,
}).validate())
}
})
}

0 comments on commit c53a948

Please sign in to comment.