Merge pull request #9982 from hashicorp/f-nsiso-driver

drivers/exec+java: Add configuration to restore previous PID/IPC namespace behavior

CHANGELOG.md

@@ -6,6 +6,7 @@ FEATURES:
 IMPROVEMENTS:
  * cli: Improved `scaling policy` commands with -verbose, auto-completion, and prefix-matching [[GH-9964](https://github.com/hashicorp/nomad/issues/9964)]
  * consul/connect: Made handling of sidecar task container image URLs consistent with the `docker` task driver. [[GH-9580](https://github.com/hashicorp/nomad/issues/9580)]
+ * drivers/exec+java: Added client plugin configuration to re-enable previous PID/IPC namespace behavior [[GH-9982](https://github.com/hashicorp/nomad/pull/9982)]
 
 BUG FIXES:
  * consul: Fixed a bug where failing tasks with group services would only cause the allocation to restart once instead of respecting the `restart` field. [[GH-9869](https://github.com/hashicorp/nomad/issues/9869)]

drivers/exec/driver.go

@@ -65,6 +65,14 @@ var (
 			hclspec.NewAttr("no_pivot_root", "bool", false),
 			hclspec.NewLiteral("false"),
 		),
+		"default_pid_mode": hclspec.NewDefault(
+			hclspec.NewAttr("default_pid_mode", "string", false),
+			hclspec.NewLiteral(`"private"`),
+		),
+		"default_ipc_mode": hclspec.NewDefault(
+			hclspec.NewAttr("default_ipc_mode", "string", false),
+			hclspec.NewLiteral(`"private"`),
+		),
 	})
 
 	// taskConfigSpec is the hcl specification for the driver config section of
@@ -122,6 +130,30 @@ type Config struct {
 	// NoPivotRoot disables the use of pivot_root, useful when the root partition
 	// is on ramdisk
 	NoPivotRoot bool `codec:"no_pivot_root"`
+
+	// DefaultModePID is the default PID isolation set for all tasks using
+	// exec-based task drivers.
+	DefaultModePID string `codec:"default_pid_mode"`
+
+	// DefaultModeIPC is the default IPC isolation set for all tasks using
+	// exec-based task drivers.
+	DefaultModeIPC string `codec:"default_ipc_mode"`
+}
+
+func (c *Config) validate() error {
+	switch c.DefaultModePID {
+	case executor.IsolationModePrivate, executor.IsolationModeHost:
+	default:
+		return fmt.Errorf("default_pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModePID)
+	}
+
+	switch c.DefaultModeIPC {
+	case executor.IsolationModePrivate, executor.IsolationModeHost:
+	default:
+		return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
+	}
+
+	return nil
 }
 
 // TaskConfig is the driver configuration of a task within a job
@@ -182,14 +214,18 @@ func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
 }
 
 func (d *Driver) SetConfig(cfg *base.Config) error {
+	// unpack, validate, and set agent plugin config
 	var config Config
 	if len(cfg.PluginConfig) != 0 {
 		if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
 			return err
 		}
 	}
-
+	if err := config.validate(); err != nil {
+		return err
+	}
 	d.config = config
+
 	if cfg != nil && cfg.AgentConfig != nil {
 		d.nomadConfig = cfg.AgentConfig.Driver
 	}
@@ -383,6 +419,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		Mounts:           cfg.Mounts,
 		Devices:          cfg.Devices,
 		NetworkIsolation: cfg.NetworkIsolation,
+		DefaultModePID:   d.config.DefaultModePID,
+		DefaultModeIPC:   d.config.DefaultModeIPC,
 	}
 
 	ps, err := exec.Launch(execCmd)

drivers/exec/driver_test.go

@@ -3,6 +3,7 @@ package exec
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -16,6 +17,7 @@ import (
 	"time"
 
 	ctestutils "github.com/hashicorp/nomad/client/testutil"
+	"github.com/hashicorp/nomad/drivers/shared/executor"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/helper/testtask"
@@ -273,7 +275,7 @@ func TestExecDriver_StartWaitRecover(t *testing.T) {
 // task dies, the orphans in the PID namespaces are killed by the kernel
 func TestExecDriver_NoOrphans(t *testing.T) {
 	t.Parallel()
-	require := require.New(t)
+	r := require.New(t)
 	ctestutils.ExecCompatible(t)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -283,6 +285,17 @@ func TestExecDriver_NoOrphans(t *testing.T) {
 	harness := dtestutil.NewDriverHarness(t, d)
 	defer harness.Kill()
 
+	config := &Config{
+		NoPivotRoot:    false,
+		DefaultModePID: executor.IsolationModePrivate,
+		DefaultModeIPC: executor.IsolationModePrivate,
+	}
+
+	var data []byte
+	r.NoError(basePlug.MsgPackEncode(&data, config))
+	baseConfig := &basePlug.Config{PluginConfig: data}
+	r.NoError(harness.SetConfig(baseConfig))
+
 	task := &drivers.TaskConfig{
 		ID:   uuid.Generate(),
 		Name: "test",
@@ -295,21 +308,21 @@ func TestExecDriver_NoOrphans(t *testing.T) {
 	taskConfig["command"] = "/bin/sh"
 	// print the child PID in the task PID namespace, then sleep for 5 seconds to give us a chance to examine processes
 	taskConfig["args"] = []string{"-c", fmt.Sprintf(`sleep 3600 & sleep 20`)}
-	require.NoError(task.EncodeConcreteDriverConfig(&taskConfig))
+	r.NoError(task.EncodeConcreteDriverConfig(&taskConfig))
 
 	handle, _, err := harness.StartTask(task)
-	require.NoError(err)
+	r.NoError(err)
 	defer harness.DestroyTask(task.ID, true)
 
 	waitCh, err := harness.WaitTask(context.Background(), handle.Config.ID)
-	require.NoError(err)
+	r.NoError(err)
 
-	require.NoError(harness.WaitUntilStarted(task.ID, 1*time.Second))
+	r.NoError(harness.WaitUntilStarted(task.ID, 1*time.Second))
 
 	var childPids []int
 	taskState := TaskState{}
 	testutil.WaitForResult(func() (bool, error) {
-		require.NoError(handle.GetDriverState(&taskState))
+		r.NoError(handle.GetDriverState(&taskState))
 		if taskState.Pid == 0 {
 			return false, fmt.Errorf("task PID is zero")
 		}
@@ -331,14 +344,14 @@ func TestExecDriver_NoOrphans(t *testing.T) {
 		}
 		return true, nil
 	}, func(err error) {
-		require.NoError(err)
+		r.NoError(err)
 	})
 
 	select {
 	case result := <-waitCh:
-		require.True(result.Successful(), "command failed: %#v", result)
+		r.True(result.Successful(), "command failed: %#v", result)
 	case <-time.After(30 * time.Second):
-		require.Fail("timeout waiting for task to shutdown")
+		r.Fail("timeout waiting for task to shutdown")
 	}
 
 	// isProcessRunning returns an error if process is not running
@@ -357,7 +370,7 @@ func TestExecDriver_NoOrphans(t *testing.T) {
 	}
 
 	// task should be dead
-	require.Error(isProcessRunning(taskState.Pid))
+	r.Error(isProcessRunning(taskState.Pid))
 
 	// all children should eventually be killed by OS
 	testutil.WaitForResult(func() (bool, error) {
@@ -372,7 +385,7 @@ func TestExecDriver_NoOrphans(t *testing.T) {
 		}
 		return true, nil
 	}, func(err error) {
-		require.NoError(err)
+		r.NoError(err)
 	})
 }
 
@@ -711,7 +724,7 @@ config {
 
 func TestExecDriver_NoPivotRoot(t *testing.T) {
 	t.Parallel()
-	require := require.New(t)
+	r := require.New(t)
 	ctestutils.ExecCompatible(t)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -720,11 +733,16 @@ func TestExecDriver_NoPivotRoot(t *testing.T) {
 	d := NewExecDriver(ctx, testlog.HCLogger(t))
 	harness := dtestutil.NewDriverHarness(t, d)
 
-	config := &Config{NoPivotRoot: true}
+	config := &Config{
+		NoPivotRoot:    true,
+		DefaultModePID: executor.IsolationModePrivate,
+		DefaultModeIPC: executor.IsolationModePrivate,
+	}
+
 	var data []byte
-	require.NoError(basePlug.MsgPackEncode(&data, config))
+	r.NoError(basePlug.MsgPackEncode(&data, config))
 	bconfig := &basePlug.Config{PluginConfig: data}
-	require.NoError(harness.SetConfig(bconfig))
+	r.NoError(harness.SetConfig(bconfig))
 
 	task := &drivers.TaskConfig{
 		ID:        uuid.Generate(),
@@ -738,9 +756,28 @@ func TestExecDriver_NoPivotRoot(t *testing.T) {
 		Command: "/bin/sleep",
 		Args:    []string{"100"},
 	}
-	require.NoError(task.EncodeConcreteDriverConfig(&tc))
+	r.NoError(task.EncodeConcreteDriverConfig(&tc))
 
 	handle, _, err := harness.StartTask(task)
-	require.NoError(err)
-	require.NotNil(handle)
+	r.NoError(err)
+	r.NotNil(handle)
+}
+
+func TestDriver_Config_validate(t *testing.T) {
+	for _, tc := range []struct {
+		pidMode, ipcMode string
+		exp              error
+	}{
+		{pidMode: "host", ipcMode: "host", exp: nil},
+		{pidMode: "private", ipcMode: "host", exp: nil},
+		{pidMode: "host", ipcMode: "private", exp: nil},
+		{pidMode: "private", ipcMode: "private", exp: nil},
+		{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
+		{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
+	} {
+		require.Equal(t, tc.exp, (&Config{
+			DefaultModePID: tc.pidMode,
+			DefaultModeIPC: tc.ipcMode,
+		}).validate())
+	}
 }

drivers/java/driver.go

@@ -63,7 +63,16 @@ var (
 	}
 
 	// configSpec is the hcl specification returned by the ConfigSchema RPC
-	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{})
+	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
+		"default_pid_mode": hclspec.NewDefault(
+			hclspec.NewAttr("default_pid_mode", "string", false),
+			hclspec.NewLiteral(`"private"`),
+		),
+		"default_ipc_mode": hclspec.NewDefault(
+			hclspec.NewAttr("default_ipc_mode", "string", false),
+			hclspec.NewLiteral(`"private"`),
+		),
+	})
 
 	// taskConfigSpec is the hcl specification for the driver config section of
 	// a taskConfig within a job. It is returned in the TaskConfigSchema RPC
@@ -101,6 +110,33 @@ func init() {
 	}
 }
 
+// Config is the driver configuration set by the SetConfig RPC call
+type Config struct {
+	// DefaultModePID is the default PID isolation set for all tasks using
+	// exec-based task drivers.
+	DefaultModePID string `codec:"default_pid_mode"`
+
+	// DefaultModeIPC is the default IPC isolation set for all tasks using
+	// exec-based task drivers.
+	DefaultModeIPC string `codec:"default_ipc_mode"`
+}
+
+func (c *Config) validate() error {
+	switch c.DefaultModePID {
+	case executor.IsolationModePrivate, executor.IsolationModeHost:
+	default:
+		return fmt.Errorf("default_pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModePID)
+	}
+
+	switch c.DefaultModeIPC {
+	case executor.IsolationModePrivate, executor.IsolationModeHost:
+	default:
+		return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
+	}
+
+	return nil
+}
+
 // TaskConfig is the driver configuration of a taskConfig within a job
 type TaskConfig struct {
 	Class     string   `codec:"class"`
@@ -126,6 +162,9 @@ type Driver struct {
 	// event can be broadcast to all callers
 	eventer *eventer.Eventer
 
+	// config is the driver configuration set by the SetConfig RPC
+	config Config
+
 	// tasks is the in memory datastore mapping taskIDs to taskHandle
 	tasks *taskStore
 
@@ -159,6 +198,18 @@ func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
 }
 
 func (d *Driver) SetConfig(cfg *base.Config) error {
+	// unpack, validate, and set agent plugin config
+	var config Config
+	if len(cfg.PluginConfig) != 0 {
+		if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
+			return err
+		}
+	}
+	if err := config.validate(); err != nil {
+		return err
+	}
+	d.config = config
+
 	if cfg != nil && cfg.AgentConfig != nil {
 		d.nomadConfig = cfg.AgentConfig.Driver
 	}
@@ -374,6 +425,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		Mounts:           cfg.Mounts,
 		Devices:          cfg.Devices,
 		NetworkIsolation: cfg.NetworkIsolation,
+		DefaultModePID:   d.config.DefaultModePID,
+		DefaultModeIPC:   d.config.DefaultModeIPC,
 	}
 
 	ps, err := exec.Launch(execCmd)