diff --git a/client/config/config.go b/client/config/config.go
index 3c968480ac3e..1d50c5669fe6 100644
--- a/client/config/config.go
+++ b/client/config/config.go
@@ -99,6 +99,10 @@ type Config struct {
// devices and IPs.
GloballyReservedPorts []int
+ // A mapping of directories on the host OS to attempt to embed inside each
+ // task's chroot.
+ ChrootEnv map[string]string
+
// Options provides arbitrary key-value configuration for nomad internals,
// like fingerprinters and drivers. The format is:
//
diff --git a/client/driver/exec.go b/client/driver/exec.go
index ab534b82071a..728ed3f5f74f 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -121,11 +121,12 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
return nil, err
}
executorCtx := &executor.ExecutorContext{
- TaskEnv: d.taskEnv,
- Driver: "exec",
- AllocDir: ctx.AllocDir,
- AllocID: ctx.AllocID,
- Task: task,
+ TaskEnv: d.taskEnv,
+ Driver: "exec",
+ AllocDir: ctx.AllocDir,
+ AllocID: ctx.AllocID,
+ ChrootEnv: d.config.ChrootEnv,
+ Task: task,
}
ps, err := exec.LaunchCmd(&executor.ExecCommand{
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 7685bfdf25c2..7fc973e63074 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -102,6 +102,10 @@ type ExecutorContext struct {
// AllocID is the allocation id to which the task belongs
AllocID string
+ // A mapping of directories on the host OS to attempt to embed inside each
+ // task's chroot.
+ ChrootEnv map[string]string
+
// Driver is the name of the driver that invoked the executor
Driver string
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 1f6fef13cc4b..c673555cc5f2 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -227,7 +227,12 @@ func (e *UniversalExecutor) configureChroot() error {
return err
}
- if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
+ chroot := chrootEnv
+ if len(e.ctx.ChrootEnv) > 0 {
+ chroot = e.ctx.ChrootEnv
+ }
+
+ if err := allocDir.Embed(e.ctx.Task.Name, chroot); err != nil {
return err
}
diff --git a/client/driver/executor/executor_linux_test.go b/client/driver/executor/executor_linux_test.go
index 9af75dec51d0..c3006071a4f8 100644
--- a/client/driver/executor/executor_linux_test.go
+++ b/client/driver/executor/executor_linux_test.go
@@ -9,15 +9,38 @@ import (
"strings"
"testing"
+ "github.com/hashicorp/nomad/client/driver/env"
cstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/testutil"
+ "github.com/hashicorp/nomad/nomad/mock"
)
+func testExecutorContextWithChroot(t *testing.T) *ExecutorContext {
+ taskEnv := env.NewTaskEnvironment(mock.Node())
+ task, allocDir := mockAllocDir(t)
+ ctx := &ExecutorContext{
+ TaskEnv: taskEnv,
+ Task: task,
+ AllocDir: allocDir,
+ ChrootEnv: map[string]string{
+ "/etc/ld.so.cache": "/etc/ld.so.cache",
+ "/etc/ld.so.conf": "/etc/ld.so.conf",
+ "/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
+ "/lib": "/lib",
+ "/lib64": "/lib64",
+ "/usr/lib": "/usr/lib",
+ "/bin/ls": "/bin/ls",
+ "/foobar": "/does/not/exist",
+ },
+ }
+ return ctx
+}
+
func TestExecutor_IsolationAndConstraints(t *testing.T) {
testutil.ExecCompatible(t)
- execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}}
- ctx := testExecutorContext(t)
+ execCmd := ExecCommand{Cmd: "/bin/ls", Args: []string{"-F", "/", "/etc/"}}
+ ctx := testExecutorContextWithChroot(t)
defer ctx.AllocDir.Destroy()
execCmd.FSIsolation = true
@@ -58,7 +81,7 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) {
t.Fatalf("file %v hasn't been removed", memLimits)
}
- expected := "hello world"
+ expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/"
file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0")
output, err := ioutil.ReadFile(file)
if err != nil {
diff --git a/client/driver/java.go b/client/driver/java.go
index 532199433f48..24e6ce6a8fd8 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -203,11 +203,12 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
return nil, err
}
executorCtx := &executor.ExecutorContext{
- TaskEnv: d.taskEnv,
- Driver: "java",
- AllocDir: ctx.AllocDir,
- AllocID: ctx.AllocID,
- Task: task,
+ TaskEnv: d.taskEnv,
+ Driver: "java",
+ AllocDir: ctx.AllocDir,
+ AllocID: ctx.AllocID,
+ ChrootEnv: d.config.ChrootEnv,
+ Task: task,
}
absPath, err := GetAbsolutePath("java")
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 5b5bfaa1e676..b50d89f600a5 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -274,6 +274,7 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
if a.config.Client.NetworkInterface != "" {
conf.NetworkInterface = a.config.Client.NetworkInterface
}
+ conf.ChrootEnv = a.config.Client.ChrootEnv
conf.Options = a.config.Client.Options
// Logging deprecation messages about consul related configuration in client
// options
diff --git a/command/agent/config-test-fixtures/basic.hcl b/command/agent/config-test-fixtures/basic.hcl
index 8715ef988379..2e132ddd3187 100644
--- a/command/agent/config-test-fixtures/basic.hcl
+++ b/command/agent/config-test-fixtures/basic.hcl
@@ -33,6 +33,10 @@ client {
foo = "bar"
baz = "zip"
}
+ chroot_env {
+ "/opt/myapp/etc" = "/etc"
+ "/opt/myapp/bin" = "/bin"
+ }
network_interface = "eth0"
network_speed = 100
reserved {
diff --git a/command/agent/config.go b/command/agent/config.go
index 32bfd4d42099..8a0a6e482534 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -156,6 +156,10 @@ type ClientConfig struct {
// Metadata associated with the node
Meta map[string]string `mapstructure:"meta"`
+ // A mapping of directories on the host OS to attempt to embed inside each
+ // task's chroot.
+ ChrootEnv map[string]string `mapstructure:"chroot_env"`
+
// Interface to use for network fingerprinting
NetworkInterface string `mapstructure:"network_interface"`
@@ -720,6 +724,14 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
result.Meta[k] = v
}
+ // Add the chroot_env map values
+ if result.ChrootEnv == nil {
+ result.ChrootEnv = make(map[string]string)
+ }
+ for k, v := range b.ChrootEnv {
+ result.ChrootEnv[k] = v
+ }
+
return &result
}
diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go
index b3d3dd969490..8ea5eb206af8 100644
--- a/command/agent/config_parse.go
+++ b/command/agent/config_parse.go
@@ -315,6 +315,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
"node_class",
"options",
"meta",
+ "chroot_env",
"network_interface",
"network_speed",
"max_kill_timeout",
@@ -334,6 +335,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
delete(m, "options")
delete(m, "meta")
+ delete(m, "chroot_env")
delete(m, "reserved")
delete(m, "stats")
@@ -370,6 +372,20 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
}
}
+ // Parse out chroot_env fields. These are in HCL as a list so we need to
+ // iterate over them and merge them.
+ if chrootEnvO := listVal.Filter("chroot_env"); len(chrootEnvO.Items) > 0 {
+ for _, o := range chrootEnvO.Elem().Items {
+ var m map[string]interface{}
+ if err := hcl.DecodeObject(&m, o.Val); err != nil {
+ return err
+ }
+ if err := mapstructure.WeakDecode(m, &config.ChrootEnv); err != nil {
+ return err
+ }
+ }
+ }
+
// Parse reserved config
if o := listVal.Filter("reserved"); len(o.Items) > 0 {
if err := parseReserved(&config.Reserved, o); err != nil {
diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go
index 61cf7a3b555b..0e126eac4561 100644
--- a/command/agent/config_parse_test.go
+++ b/command/agent/config_parse_test.go
@@ -53,6 +53,10 @@ func TestConfig_Parse(t *testing.T) {
"foo": "bar",
"baz": "zip",
},
+ ChrootEnv: map[string]string{
+ "/opt/myapp/etc": "/etc",
+ "/opt/myapp/bin": "/bin",
+ },
NetworkInterface: "eth0",
NetworkSpeed: 100,
MaxKillTimeout: "10s",
diff --git a/command/agent/config_test.go b/command/agent/config_test.go
index 8306012288e5..b83454bb15b7 100644
--- a/command/agent/config_test.go
+++ b/command/agent/config_test.go
@@ -139,6 +139,7 @@ func TestConfig_Merge(t *testing.T) {
"foo": "bar",
"baz": "zip",
},
+ ChrootEnv: map[string]string{},
ClientMaxPort: 20000,
ClientMinPort: 22000,
NetworkSpeed: 105,
diff --git a/website/source/docs/agent/config.html.md b/website/source/docs/agent/config.html.md
index 371c1137de7d..c28fe1b89ab1 100644
--- a/website/source/docs/agent/config.html.md
+++ b/website/source/docs/agent/config.html.md
@@ -421,6 +421,9 @@ configured on server nodes.
* `options`: This is a key/value mapping of internal
configuration for clients, such as for driver configuration. Please see
[here](#options_map) for a description of available options.
+ * `chroot_env`: This is a key/value mapping that
+ defines the chroot environment for jobs using the Exec and Java drivers.
+ Please see [here](#chroot_env_map) for an example and further information.
* `network_interface`: This is a string to force
network fingerprinting to use a specific network interface
* `network_speed`: This is an int that sets the
@@ -496,6 +499,31 @@ documentation [here](/docs/drivers/index.html)
If specified, fingerprinters not in the whitelist will be disabled. If the
whitelist is empty, all fingerprinters are used.
+### Client ChrootEnv Map
+
+Drivers based on [Isolated Fork/Exec](/docs/drivers/exec.html) implement file
+system isolation using chroot on Linux. The `chroot_env` map allows the chroot
+environment to be configured using source paths on the host operating system.
+The mapping format is: `source_path -> dest_path`.
+
+The following example specifies a chroot which contains just enough to run the
+`ls` utility, and not much else:
+
+```
+chroot_env {
+ "/bin/ls" = "/bin/ls"
+ "/etc/ld.so.cache" = "/etc/ld.so.cache"
+ "/etc/ld.so.conf" = "/etc/ld.so.conf"
+ "/etc/ld.so.conf.d" = "/etc/ld.so.conf.d"
+ "/lib" = "/lib"
+ "/lib64" = "/lib64"
+}
+```
+
+When `chroot_env` is unspecified, the `exec` driver will use a default chroot
+environment with the most commonly used parts of the operating system. See
+`exec` documentation for the full list [here](/docs/drivers/exec.html#chroot).
+
## Command-line Options
A subset of the available Nomad agent configuration can optionally be passed in
diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md
index 4477c860ec78..20dfea5f765e 100644
--- a/website/source/docs/drivers/exec.html.md
+++ b/website/source/docs/drivers/exec.html.md
@@ -96,9 +96,12 @@ the client and the configuration.
On Linux, Nomad will use cgroups, and a chroot to isolate the
resources of a process and as such the Nomad agent must be run as root.
-### Chroot
+### Chroot
The chroot is populated with data in the following folders from the host
machine:
`["/bin", "/etc", "/lib", "/lib32", "/lib64", "/run/resolvconf", "/sbin",
"/usr"]`
+
+This list is configurable through the agent client
+[configuration file](/docs/agent/config.html#chroot_env).