Merge pull request #8261 from hashicorp/f-disable-host-volume-by-default

Restrict Host filesystem access in Docker and Qemu

CHANGELOG.md

@@ -12,6 +12,11 @@ FEATURES:
  * **Monitor UI**: Stream client and agent logs from the UI just like you would with the nomad monitor CLI command. [[GH-8177](https://github.com/hashicorp/nomad/issues/8177)]
  * **Scaling UI**: Quickly adjust the count of a task group from the UI for task groups with a scaling declaration. [[GH-8207](https://github.com/hashicorp/nomad/issues/8207)]
 
+__BACKWARDS INCOMPATIBILITIES:__
+ * driver/docker: The Docker driver no longer allows binding host volumes by default.
+   Operators can set `volume` `enabled` plugin configuration to restore previous permissive behavior. [[GH-8261](https://github.com/hashicorp/nomad/issues/8261)]
+ * driver/qemu: The Qemu driver requires images to reside in a operator-defined paths allowed for task access. [[GH-8261](https://github.com/hashicorp/nomad/issues/8261)]
+
 IMPROVEMENTS:
 
 * core: Support for persisting previous task group counts when updating a job [[GH-8168](https://github.com/hashicorp/nomad/issues/8168)]

drivers/docker/config.go

@@ -237,10 +237,7 @@ var (
 		// defaulted needed for both if the volumes {...} block is not set and
 		// if the default fields are missing
 		"volumes": hclspec.NewDefault(hclspec.NewBlock("volumes", false, hclspec.NewObject(map[string]*hclspec.Spec{
-			"enabled": hclspec.NewDefault(
-				hclspec.NewAttr("enabled", "bool", false),
-				hclspec.NewLiteral("true"),
-			),
+			"enabled":      hclspec.NewAttr("enabled", "bool", false),
 			"selinuxlabel": hclspec.NewAttr("selinuxlabel", "string", false),
 		})), hclspec.NewLiteral("{ enabled = true }")),
 		"allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false),

drivers/qemu/driver.go

@@ -83,7 +83,9 @@ var (
 	}
 
 	// configSpec is the hcl specification returned by the ConfigSchema RPC
-	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{})
+	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
+		"image_paths": hclspec.NewAttr("image_paths", "list(string)", false),
+	})
 
 	// taskConfigSpec is the hcl specification for the driver config section of
 	// a taskConfig within a job. It is returned in the TaskConfigSchema RPC
@@ -126,12 +128,21 @@ type TaskState struct {
 	StartedAt      time.Time
 }
 
+// Config is the driver configuration set by SetConfig RPC call
+type Config struct {
+	// ImagePaths is an allow-list of paths qemu is allowed to load an image from
+	ImagePaths []string `codec:"image_paths"`
+}
+
 // Driver is a driver for running images via Qemu
 type Driver struct {
 	// eventer is used to handle multiplexing of TaskEvents calls such that an
 	// event can be broadcast to all callers
 	eventer *eventer.Eventer
 
+	// config is the driver configuration set by the SetConfig RPC
+	config Config
+
 	// tasks is the in memory datastore mapping taskIDs to qemuTaskHandle
 	tasks *taskStore
 
@@ -165,6 +176,14 @@ func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
 }
 
 func (d *Driver) SetConfig(cfg *base.Config) error {
+	var config Config
+	if len(cfg.PluginConfig) != 0 {
+		if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
+			return err
+		}
+	}
+
+	d.config = config
 	if cfg.AgentConfig != nil {
 		d.nomadConfig = cfg.AgentConfig.Driver
 	}
@@ -290,6 +309,31 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
 	return nil
 }
 
+func isAllowedImagePath(allowedPaths []string, allocDir, imagePath string) bool {
+	if !filepath.IsAbs(imagePath) {
+		imagePath = filepath.Join(allocDir, imagePath)
+	}
+
+	isParent := func(parent, path string) bool {
+		rel, err := filepath.Rel(parent, path)
+		return err == nil && !strings.HasPrefix(rel, "..")
+	}
+
+	// check if path is under alloc dir
+	if isParent(allocDir, imagePath) {
+		return true
+	}
+
+	// check allowed paths
+	for _, ap := range allowedPaths {
+		if isParent(ap, imagePath) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
 	if _, ok := d.tasks.Get(cfg.ID); ok {
 		return nil, nil, fmt.Errorf("taskConfig with ID '%s' already started", cfg.ID)
@@ -314,6 +358,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	}
 	vmID := filepath.Base(vmPath)
 
+	if !isAllowedImagePath(d.config.ImagePaths, cfg.AllocDir, vmPath) {
+		return nil, nil, fmt.Errorf("image_path is not in the allowed paths")
+	}
+
 	// Parse configuration arguments
 	// Create the base arguments
 	accelerator := "tcg"

drivers/qemu/driver_test.go

@@ -424,3 +424,32 @@ config {
 
 	require.EqualValues(t, expected, tc)
 }
+
+func TestIsAllowedImagePath(t *testing.T) {
+	allowedPaths := []string{"/tmp", "/opt/qemu"}
+	allocDir := "/opt/nomad/some-alloc-dir"
+
+	validPaths := []string{
+		"local/path",
+		"/tmp/subdir/qemu-image",
+		"/opt/qemu/image",
+		"/opt/qemu/subdir/image",
+		"/opt/nomad/some-alloc-dir/local/image.img",
+	}
+
+	invalidPaths := []string{
+		"/image.img",
+		"../image.img",
+		"/tmpimage.img",
+		"/opt/other/image.img",
+		"/opt/nomad-submatch.img",
+	}
+
+	for _, p := range validPaths {
+		require.Truef(t, isAllowedImagePath(allowedPaths, allocDir, p), "path should be allowed: %v", p)
+	}
+
+	for _, p := range invalidPaths {
+		require.Falsef(t, isAllowedImagePath(allowedPaths, allocDir, p), "path should be not allowed: %v", p)
+	}
+}

website/pages/docs/drivers/docker.mdx

@@ -780,7 +780,7 @@ plugin "docker" {
 
 - `volumes` stanza:
 
-  - `enabled` - Defaults to `true`. Allows tasks to bind host paths
+  - `enabled` - Defaults to `false`. Allows tasks to bind host paths
     (`volumes`) inside their container and use volume drivers
     (`volume_driver`). Binding relative paths is always allowed and will be
     resolved relative to the allocation's directory.
@@ -844,7 +844,7 @@ options](/docs/configuration/client#options):
   deleting it. If a tasks is received that uses the same image within the delay,
   the image will be reused.
 
-- `docker.volumes.enabled`: Defaults to `true`. Allows tasks to bind host paths
+- `docker.volumes.enabled`: Defaults to `false`. Allows tasks to bind host paths
   (`volumes`) inside their container and use volume drivers (`volume_driver`).
   Binding relative paths is always allowed and will be resolved relative to the
   allocation's directory.

website/pages/docs/drivers/qemu.mdx

@@ -128,6 +128,19 @@ job "docs" {
 }
 ```
 
+## Plugin Options
+
+```hcl
+plugin "qemu" {
+  config {
+    image_paths = ["/mnt/image/paths"]
+  }
+}
+```
+
+- `image_paths` (`[]string`: `[]`) - Specifies the host paths the Qemu driver is
+  allowed to load images from.
+
 ## Resource Isolation
 
 Nomad uses Qemu to provide full software virtualization for virtual machine

website/pages/docs/upgrade/upgrade-specific.mdx

@@ -15,6 +15,50 @@ details provided for their upgrades as a result of new features or changed
 behavior. This page is used to document those details separately from the
 standard upgrade flow.
 
+## Nomad 0.12.0
+
+### Docker access host filesystem
+
+Nomad 0.12.0 disables Docker tasks access to the host filesystem, by default.
+Prior to Nomad 0.12, Docker tasks may mount and then manipulate any host file
+and may pose a security risk.
+
+Operators now must explicitly allow tasks to access host filesystem. [Host
+Volumes](/docs/configuration/client#host_volume-stanza) provide a fine tune
+access to individual paths.
+
+To restore pre-0.12.0 behavior, you can enable [Docker
+`volume`](/docs/drivers/docker#enabled-1) to allow binding host paths, by adding
+the following to the nomad client config file:
+
+```hcl
+plugin "docker" {
+  config {
+    volume {
+      enabled = true
+    }
+  }
+}
+```
+
+### Qemu images
+
+Nomad 0.12.0 restricts the paths the Qemu tasks can load an image from. A Qemu
+task may download an image to the allocation directory to load. But images
+outside the allocation directories must be explicitly allowed by operators in
+the client agent configuration file.
+
+For example, you may allow loading Qemu images from `/mnt/qemu-images` by
+adding the following to the agent configuration file:
+
+```hcl
+plugin "qemu" {
+  config {
+    image_paths = ["/mnt/qemu-images"]
+  }
+}
+```
+
 ## Nomad 0.11.3
 
 Nomad 0.11.3 fixes a critical bug causing the nomad agent to become