update template and artifact interpolation to use client-relative paths

resolves #9839
resolves #6929
resolves #6910

client/allocrunner/taskrunner/artifact_hook.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/getter"
 	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
+	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -52,7 +53,18 @@ func (h *artifactHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 
 		h.logger.Debug("downloading artifact", "artifact", artifact.GetterSource)
 		//XXX add ctx to GetArtifact to allow cancelling long downloads
-		if err := getter.GetArtifact(req.TaskEnv, artifact, req.TaskDir.Dir); err != nil {
+		clientEnv := make(map[string]string, len(req.TaskEnv.EnvMap))
+		for k, v := range req.TaskEnv.EnvMap {
+			clientEnv[k] = v
+		}
+		clientEnv[taskenv.AllocDir] = req.TaskDir.SharedAllocDir
+		clientEnv[taskenv.TaskLocalDir] = req.TaskDir.Dir
+		clientTaskEnv := taskenv.NewTaskEnv(
+			clientEnv,
+			req.TaskEnv.DeviceEnv(),
+			req.TaskEnv.NodeAttrs,
+		)
+		if err := getter.GetArtifact(req.TaskEnv, clientTaskEnv, artifact, req.TaskDir); err != nil {
 			wrapped := structs.NewRecoverableError(
 				fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err),
 				true,

client/allocrunner/taskrunner/getter/getter.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 
 	gg "github.com/hashicorp/go-getter"
+
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
@@ -129,19 +131,36 @@ func getHeaders(env EnvReplacer, m map[string]string) http.Header {
 	return headers
 }
 
+// checkEscape returns true if the absolute path testPath escapes all of the
+// absolute paths in sandboxPaths.
+// otherwise, it returns false, indicating that testPath is part of one of the
+// acceptable sandbox paths.
+func checkEscape(testPath string, sandboxPaths []string) bool {
+	for _, p := range sandboxPaths {
+		if !helper.PathEscapesSandbox(p, testPath) {
+			return false
+		}
+	}
+	return true
+}
+
 // GetArtifact downloads an artifact into the specified task directory.
-func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir string) error {
-	ggURL, err := getGetterUrl(taskEnv, artifact)
+// clientTaskEnv is used for interpolating the destination directory of the artifact in the client
+// workloadTaskEnv is used for other interpolation operations
+func GetArtifact(workloadTaskEnv, clientTaskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir *allocdir.TaskDir) error {
+	ggURL, err := getGetterUrl(workloadTaskEnv, artifact)
 	if err != nil {
 		return newGetError(artifact.GetterSource, err, false)
 	}
 
 	// Verify the destination is still in the task sandbox after interpolation
-	// Note: we *always* join here even if we get passed an absolute path so
-	// that $NOMAD_SECRETS_DIR and friends can be used and always fall inside
-	// the task working directory
-	dest := filepath.Join(taskDir, artifact.RelativeDest)
-	escapes := helper.PathEscapesSandbox(taskDir, dest)
+	dest := clientTaskEnv.ReplaceEnv(artifact.RelativeDest)
+	// if it was a relative path (like 'local' or '../alloc', join it with the task working directory)
+	if !filepath.IsAbs(dest) {
+		dest = filepath.Join(taskDir.Dir, dest)
+	}
+	dest = filepath.Clean(dest)
+	escapes := checkEscape(dest, []string{taskDir.Dir, taskDir.SharedAllocDir})
 	if escapes {
 		return newGetError(artifact.RelativeDest,
 			errors.New("artifact destination path escapes the alloc directory"), false)
@@ -156,7 +175,7 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st
 		mode = gg.ClientModeDir
 	}
 
-	headers := getHeaders(taskEnv, artifact.GetterHeaders)
+	headers := getHeaders(workloadTaskEnv, artifact.GetterHeaders)
 	if err := getClient(ggURL, headers, mode, dest).Get(); err != nil {
 		return newGetError(ggURL, err, true)
 	}

client/allocrunner/taskrunner/task_dir_hook.go

@@ -44,7 +44,7 @@ func (h *taskDirHook) Name() string {
 func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
 	fsi := h.runner.driverCapabilities.FSIsolation
 	if v, ok := req.PreviousState[TaskDirHookIsDoneDataKey]; ok && v == "true" {
-		setEnvvars(h.runner.envBuilder, fsi, h.runner.taskDir, h.runner.clientConfig)
+		SetEnvvars(h.runner.envBuilder, fsi, h.runner.taskDir, h.runner.clientConfig)
 		resp.State = map[string]string{
 			TaskDirHookIsDoneDataKey: "true",
 		}
@@ -67,15 +67,20 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart
 	}
 
 	// Update the environment variables based on the built task directory
-	setEnvvars(h.runner.envBuilder, fsi, h.runner.taskDir, h.runner.clientConfig)
+	SetEnvvars(h.runner.envBuilder, fsi, h.runner.taskDir, h.runner.clientConfig)
 	resp.State = map[string]string{
 		TaskDirHookIsDoneDataKey: "true",
 	}
 	return nil
 }
 
-// setEnvvars sets path and host env vars depending on the FS isolation used.
-func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) {
+// SetEnvvars sets path and host env vars depending on the FS isolation used.
+func SetEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) {
+
+	envBuilder.SetClientAllocDir(taskDir.SharedAllocDir)
+	envBuilder.SetClientTaskLocalDir(taskDir.LocalDir)
+	envBuilder.SetClientSecretDir(taskDir.SecretsDir)
+
 	// Set driver-specific environment variables
 	switch fsi {
 	case drivers.FSIsolationNone:

client/allocrunner/taskrunner/template/template.go

@@ -18,7 +18,6 @@ import (
 	"github.com/hashicorp/consul-template/signals"
 	envparse "github.com/hashicorp/go-envparse"
 	multierror "github.com/hashicorp/go-multierror"
-	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/taskenv"
@@ -96,6 +95,9 @@ type TaskTemplateManagerConfig struct {
 	// TaskDir is the task's directory
 	TaskDir string
 
+	// AllocDir is the shared alloc directory
+	SharedAllocDir string
+
 	// EnvBuilder is the environment variable builder for the task.
 	EnvBuilder *taskenv.Builder
 
@@ -211,7 +213,7 @@ func (tm *TaskTemplateManager) run() {
 	}
 
 	// Read environment variables from env templates before we unblock
-	envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
+	envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder)
 	if err != nil {
 		tm.config.Lifecycle.Kill(context.Background(),
 			structs.NewTaskEvent(structs.TaskKilling).
@@ -416,7 +418,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 		}
 
 		// Read environment variables from templates
-		envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
+		envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder)
 		if err != nil {
 			tm.config.Lifecycle.Kill(context.Background(),
 				structs.NewTaskEvent(structs.TaskKilling).
@@ -565,23 +567,20 @@ func maskProcessEnv(env map[string]string) map[string]string {
 	return env
 }
 
+func (c *TaskTemplateManagerConfig) checkEscape(test string) bool {
+	for _, p := range []string{c.SharedAllocDir, c.TaskDir} {
+		if !helper.PathEscapesSandbox(p, test) {
+			return false
+		}
+	}
+	return true
+}
+
 // parseTemplateConfigs converts the tasks templates in the config into
 // consul-templates
 func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.TemplateConfig]*structs.Template, error) {
 	sandboxEnabled := !config.ClientConfig.TemplateConfig.DisableSandbox
-	taskEnv := config.EnvBuilder.Build()
-
-	// Make NOMAD_{ALLOC,TASK,SECRETS}_DIR relative paths to avoid treating
-	// them as sandbox escapes when using containers.
-	if taskEnv.EnvMap[taskenv.AllocDir] == allocdir.SharedAllocContainerPath {
-		taskEnv.EnvMap[taskenv.AllocDir] = allocdir.SharedAllocName
-	}
-	if taskEnv.EnvMap[taskenv.TaskLocalDir] == allocdir.TaskLocalContainerPath {
-		taskEnv.EnvMap[taskenv.TaskLocalDir] = allocdir.TaskLocal
-	}
-	if taskEnv.EnvMap[taskenv.SecretsDir] == allocdir.TaskSecretsContainerPath {
-		taskEnv.EnvMap[taskenv.SecretsDir] = allocdir.TaskSecrets
-	}
+	taskEnv := config.EnvBuilder.BuildClient()
 
 	ctmpls := make(map[*ctconf.TemplateConfig]*structs.Template, len(config.Templates))
 	for _, tmpl := range config.Templates {
@@ -590,22 +589,21 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa
 			src = taskEnv.ReplaceEnv(tmpl.SourcePath)
 			if !filepath.IsAbs(src) {
 				src = filepath.Join(config.TaskDir, src)
-			} else {
-				src = filepath.Clean(src)
 			}
-			escapes := helper.PathEscapesSandbox(config.TaskDir, src)
+			src = filepath.Clean(src)
+			escapes := config.checkEscape(src)
 			if escapes && sandboxEnabled {
 				return nil, sourceEscapesErr
 			}
 		}
 
 		if tmpl.DestPath != "" {
 			dest = taskEnv.ReplaceEnv(tmpl.DestPath)
-			// Note: we *always* join here even if we get passed an absolute
-			// path so that $NOMAD_SECRETS_DIR and friends can be used and
-			// always fall inside the task working directory
-			dest = filepath.Join(config.TaskDir, dest)
-			escapes := helper.PathEscapesSandbox(config.TaskDir, dest)
+			if !filepath.IsAbs(dest) {
+				dest = filepath.Join(config.TaskDir, dest)
+			}
+			dest = filepath.Clean(dest)
+			escapes := config.checkEscape(dest)
 			if escapes && sandboxEnabled {
 				return nil, destEscapesErr
 			}
@@ -740,14 +738,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 }
 
 // loadTemplateEnv loads task environment variables from all templates.
-func loadTemplateEnv(tmpls []*structs.Template, taskDir string, taskEnv *taskenv.TaskEnv) (map[string]string, error) {
+func loadTemplateEnv(tmpls []*structs.Template, taskDir string, envBuilder *taskenv.Builder) (map[string]string, error) {
+	clientEnv := envBuilder.BuildClient()
 	all := make(map[string]string, 50)
 	for _, t := range tmpls {
 		if !t.Envvars {
 			continue
 		}
 
-		dest := filepath.Join(taskDir, taskEnv.ReplaceEnv(t.DestPath))
+		dest := filepath.Join(taskDir, clientEnv.ReplaceEnv(t.DestPath))
 		f, err := os.Open(dest)
 		if err != nil {
 			return nil, fmt.Errorf("error opening env template: %v", err)

client/allocrunner/taskrunner/template_hook.go

@@ -52,6 +52,9 @@ type templateHook struct {
 
 	// taskDir is the task directory
 	taskDir string
+
+	// shared alloc dir is the shared task alloc dir
+	sharedAllocDir string
 }
 
 func newTemplateHook(config *templateHookConfig) *templateHook {
@@ -77,6 +80,7 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 
 	// Store the current Vault token and the task directory
 	h.taskDir = req.TaskDir.Dir
+	h.sharedAllocDir = req.TaskDir.SharedAllocDir
 	h.vaultToken = req.VaultToken
 
 	// Set vault namespace if specified
@@ -109,6 +113,7 @@ func (h *templateHook) newManager() (unblock chan struct{}, err error) {
 		VaultToken:           h.vaultToken,
 		VaultNamespace:       h.vaultNamespace,
 		TaskDir:              h.taskDir,
+		SharedAllocDir:       h.sharedAllocDir,
 		EnvBuilder:           h.config.envBuilder,
 		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
 	})

client/taskenv/env.go

@@ -316,6 +316,15 @@ type Builder struct {
 	// secretsDir from task's perspective; eg /secrets
 	secretsDir string
 
+	// clientAllocDir is the alloc dir from the client's perspective; eg, <data-dir>/allocs/<allod-ic>/alloc
+	clientAllocDir string
+
+	// clientLocalDir is the local dir from the client's perspective; eg <data-dir>/allocs/<task>/local
+	clientLocalDir string
+
+	// clientSecrets is the secrets dir from the client's perspective; eg <data-dir>/allocs/<task>/secrets
+	clientSecretsDir string
+
 	cpuLimit         int64
 	memLimit         int64
 	taskName         string
@@ -525,6 +534,23 @@ func (b *Builder) Build() *TaskEnv {
 	return NewTaskEnv(cleanedEnv, deviceEnvs, nodeAttrs)
 }
 
+// BuildClient builds environment variables for interpolation, but with client-relative
+// paths for NOMAD_*_DIR.
+// BuildClient must be called after all the tasks environment values have been set.
+func (b *Builder) BuildClient() *TaskEnv {
+	env := b.Build()
+	if b.clientAllocDir != "" {
+		env.EnvMap[AllocDir] = b.clientAllocDir
+	}
+	if b.clientLocalDir != "" {
+		env.EnvMap[TaskLocalDir] = b.clientLocalDir
+	}
+	if b.clientSecretsDir != "" {
+		env.EnvMap[SecretsDir] = b.clientSecretsDir
+	}
+	return env
+}
+
 // Update task updates the environment based on a new alloc and task.
 func (b *Builder) UpdateTask(alloc *structs.Allocation, task *structs.Task) *Builder {
 	b.mu.Lock()
@@ -726,6 +752,27 @@ func (b *Builder) SetTaskLocalDir(dir string) *Builder {
 	return b
 }
 
+func (b *Builder) SetClientAllocDir(dir string) *Builder {
+	b.mu.Lock()
+	b.clientAllocDir = dir
+	b.mu.Unlock()
+	return b
+}
+
+func (b *Builder) SetClientTaskLocalDir(dir string) *Builder {
+	b.mu.Lock()
+	b.clientLocalDir = dir
+	b.mu.Unlock()
+	return b
+}
+
+func (b *Builder) SetClientSecretDir(dir string) *Builder {
+	b.mu.Lock()
+	b.clientSecretsDir = dir
+	b.mu.Unlock()
+	return b
+}
+
 func (b *Builder) SetSecretsDir(dir string) *Builder {
 	b.mu.Lock()
 	b.secretsDir = dir

e2e/consultemplate/consultemplate.go

@@ -297,6 +297,58 @@ func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_Bad(f *framework.F)
 	f.True(found, "alloc failed but NOT due to expected source path escape error")
 }
 
+// TestTemplatePathInterpolation_SharedAlloc asserts that NOMAD_ALLOC_DIR
+// is supported as a destination for artifact and template blocks, and
+// that it is properly interpolated for task drivers with varying
+// filesystem isolation
+func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_SharedAllocDir(f *framework.F) {
+	jobID := "template-shared-alloc-" + uuid.Generate()[:8]
+	tc.jobIDs = append(tc.jobIDs, jobID)
+
+	allocStubs := e2eutil.RegisterAndWaitForAllocs(
+		f.T(), tc.Nomad(), "consultemplate/input/template_shared_alloc.nomad", jobID, "")
+	f.Len(allocStubs, 1)
+	allocID := allocStubs[0].ID
+
+	e2eutil.WaitForAllocRunning(f.T(), tc.Nomad(), allocID)
+
+	for _, task := range []string{"raw_exec", "docker", "exec"} {
+		f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/raw_exec.env",
+			func(out string) bool {
+				return len(out) > 0 && strings.TrimSpace(out) != "/alloc"
+			}, nil), "expected raw_exec.env to not be '/alloc'")
+
+		f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/exec.env",
+			func(out string) bool {
+				return strings.TrimSpace(out) == "/alloc"
+			}, nil), "expected shared exec.env to contain '/alloc'")
+
+		f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/docker.env",
+			func(out string) bool {
+				return strings.TrimSpace(out) == "/alloc"
+			}, nil), "expected shared docker.env to contain '/alloc'")
+	}
+}
+
+func waitForTaskFile(allocID, task, path string, test func(out string) bool, wc *e2e.WaitConfig) error {
+	var err error
+	var out string
+	interval, retries := wc.OrDefault()
+
+	testutil.WaitForResultRetries(retries, func() (bool, error) {
+		time.Sleep(interval)
+		out, err = e2e.Command("nomad", "alloc", "exec", "-task", task, allocID, "sh", "-c", "cat "+path)
+		if err != nil {
+			return false, fmt.Errorf("could not cat file %q from task %q in allocation %q: %v",
+				path, task, allocID, err)
+		}
+		return test(out), nil
+	}, func(e error) {
+		err = fmt.Errorf("test for file content failed: got %#v\nerror: %v", out, e)
+	})
+	return err
+}
+
 // waitForTemplateRender is a helper that grabs a file via alloc fs
 // and tests it for
 func waitForTemplateRender(allocID, path string, test func(string) bool, wc *e2e.WaitConfig) error {