WIP: better interpolation support for template and artifact source/destination

client/allocrunner/taskrunner/artifact_hook.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/getter"
 	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
+	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -52,7 +53,20 @@ func (h *artifactHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 
 		h.logger.Debug("downloading artifact", "artifact", artifact.GetterSource)
 		//XXX add ctx to GetArtifact to allow cancelling long downloads
-		if err := getter.GetArtifact(req.TaskEnv, artifact, req.TaskDir.Dir); err != nil {
+		clientEnv := make(map[string]string, len(req.TaskEnv.EnvMap))
+		for k, v := range req.TaskEnv.EnvMap {
+			clientEnv[k] = v
+		}
+		clientEnv[taskenv.AllocDir] = req.TaskDir.SharedAllocDir
+		clientEnv[taskenv.TaskLocalDir] = req.TaskDir.Dir
+		clientTaskEnv := taskenv.NewTaskEnv(
+			clientEnv,
+			req.TaskEnv.DeviceEnv(),
+			req.TaskEnv.NodeAttrs,
+		)
+		if err := getter.GetArtifact(req.TaskEnv, clientTaskEnv, artifact,
+			req.TaskDir.Dir, req.TaskDir.SharedAllocDir); err != nil {
+
 			wrapped := structs.NewRecoverableError(
 				fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err),
 				true,

client/allocrunner/taskrunner/getter/getter.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 
 	gg "github.com/hashicorp/go-getter"
+
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
@@ -129,19 +130,36 @@ func getHeaders(env EnvReplacer, m map[string]string) http.Header {
 	return headers
 }
 
+// checkEscape returns true if the absolute path testPath escapes all of the
+// absolute paths in sandboxPaths.
+// otherwise, it returns false, indicating that testPath is part of one of the
+// acceptable sandbox paths.
+func checkEscape(testPath string, sandboxPaths []string) bool {
+	for _, p := range sandboxPaths {
+		if !helper.PathEscapesSandbox(p, testPath) {
+			return false
+		}
+	}
+	return true
+}
+
 // GetArtifact downloads an artifact into the specified task directory.
-func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir string) error {
-	ggURL, err := getGetterUrl(taskEnv, artifact)
+// clientTaskEnv is used for interpolating the destination directory of the artifact in the client
+// workloadTaskEnv is used for other interpolation operations
+func GetArtifact(workloadTaskEnv, clientTaskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir, sharedAllocDir string) error {
+	ggURL, err := getGetterUrl(workloadTaskEnv, artifact)
 	if err != nil {
 		return newGetError(artifact.GetterSource, err, false)
 	}
 
 	// Verify the destination is still in the task sandbox after interpolation
-	// Note: we *always* join here even if we get passed an absolute path so
-	// that $NOMAD_SECRETS_DIR and friends can be used and always fall inside
-	// the task working directory
-	dest := filepath.Join(taskDir, artifact.RelativeDest)
-	escapes := helper.PathEscapesSandbox(taskDir, dest)
+	dest := clientTaskEnv.ReplaceEnv(artifact.RelativeDest)
+	// if it was a relative path (like 'local' or '../alloc', join it with the task working directory)
+	if !filepath.IsAbs(dest) {
+		dest = filepath.Join(taskDir, dest)
+	}
+	dest = filepath.Clean(dest)
+	escapes := checkEscape(dest, []string{taskDir, sharedAllocDir})
 	if escapes {
 		return newGetError(artifact.RelativeDest,
 			errors.New("artifact destination path escapes the alloc directory"), false)
@@ -156,7 +174,7 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st
 		mode = gg.ClientModeDir
 	}
 
-	headers := getHeaders(taskEnv, artifact.GetterHeaders)
+	headers := getHeaders(workloadTaskEnv, artifact.GetterHeaders)
 	if err := getClient(ggURL, headers, mode, dest).Get(); err != nil {
 		return newGetError(ggURL, err, true)
 	}

client/allocrunner/taskrunner/getter/getter_test.go

@@ -95,7 +95,7 @@ func TestGetArtifact_Headers(t *testing.T) {
 
 	// Download the artifact.
 	taskEnv := new(upperReplacer)
-	err = GetArtifact(taskEnv, artifact, taskDir)
+	err = GetArtifact(taskEnv, taskEnv, artifact, taskDir, "")
 	require.NoError(t, err)
 
 	// Verify artifact exists.
@@ -129,7 +129,7 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) {
 	}
 
 	// Download the artifact
-	if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
+	if err := GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, ""); err != nil {
 		t.Fatalf("GetArtifact failed: %v", err)
 	}
 
@@ -163,7 +163,7 @@ func TestGetArtifact_File_RelativeDest(t *testing.T) {
 	}
 
 	// Download the artifact
-	if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
+	if err := GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, ""); err != nil {
 		t.Fatalf("GetArtifact failed: %v", err)
 	}
 
@@ -197,7 +197,7 @@ func TestGetArtifact_File_EscapeDest(t *testing.T) {
 	}
 
 	// attempt to download the artifact
-	err = GetArtifact(noopTaskEnv, artifact, taskDir)
+	err = GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, "")
 	if err == nil || !strings.Contains(err.Error(), "escapes") {
 		t.Fatalf("expected GetArtifact to disallow sandbox escape: %v", err)
 	}
@@ -247,7 +247,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) {
 	}
 
 	// Download the artifact and expect an error
-	if err := GetArtifact(noopTaskEnv, artifact, taskDir); err == nil {
+	if err := GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, ""); err == nil {
 		t.Fatalf("GetArtifact should have failed")
 	}
 }
@@ -312,7 +312,7 @@ func TestGetArtifact_Archive(t *testing.T) {
 		},
 	}
 
-	if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
+	if err := GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, ""); err != nil {
 		t.Fatalf("GetArtifact failed: %v", err)
 	}
 
@@ -345,7 +345,7 @@ func TestGetArtifact_Setuid(t *testing.T) {
 		},
 	}
 
-	require.NoError(t, GetArtifact(noopTaskEnv, artifact, taskDir))
+	require.NoError(t, GetArtifact(noopTaskEnv, noopTaskEnv, artifact, taskDir, ""))
 
 	var expected map[string]int
 

client/allocrunner/taskrunner/task_dir_hook.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	log "github.com/hashicorp/go-hclog"
+
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	cconfig "github.com/hashicorp/nomad/client/config"
@@ -76,6 +77,11 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart
 
 // setEnvvars sets path and host env vars depending on the FS isolation used.
 func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) {
+
+	envBuilder.SetClientAllocDir(taskDir.SharedAllocDir)
+	envBuilder.SetClientTaskLocalDir(taskDir.LocalDir)
+	envBuilder.SetClientSecretDir(taskDir.SecretsDir)
+
 	// Set driver-specific environment variables
 	switch fsi {
 	case drivers.FSIsolationNone:
@@ -93,11 +99,10 @@ func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *a
 	// Set the host environment variables for non-image based drivers
 	if fsi != drivers.FSIsolationImage {
 		// COMPAT(1.0) using inclusive language, blacklist is kept for backward compatibility.
-		denylist := conf.ReadAlternativeDefault(
+		filter := strings.Split(conf.ReadAlternativeDefault(
 			[]string{"env.denylist", "env.blacklist"},
 			cconfig.DefaultEnvDenylist,
-		)
-		filter := strings.Split(denylist, ",")
+		), ",")
 		envBuilder.SetHostEnvvars(filter)
 	}
 }

client/allocrunner/taskrunner/template/template.go

@@ -18,7 +18,6 @@ import (
 	"github.com/hashicorp/consul-template/signals"
 	envparse "github.com/hashicorp/go-envparse"
 	multierror "github.com/hashicorp/go-multierror"
-	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/taskenv"
@@ -96,6 +95,9 @@ type TaskTemplateManagerConfig struct {
 	// TaskDir is the task's directory
 	TaskDir string
 
+	// AllocDir is the shared alloc directory
+	SharedAllocDir string
+
 	// EnvBuilder is the environment variable builder for the task.
 	EnvBuilder *taskenv.Builder
 
@@ -211,7 +213,7 @@ func (tm *TaskTemplateManager) run() {
 	}
 
 	// Read environment variables from env templates before we unblock
-	envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
+	envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder)
 	if err != nil {
 		tm.config.Lifecycle.Kill(context.Background(),
 			structs.NewTaskEvent(structs.TaskKilling).
@@ -416,7 +418,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 		}
 
 		// Read environment variables from templates
-		envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
+		envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder)
 		if err != nil {
 			tm.config.Lifecycle.Kill(context.Background(),
 				structs.NewTaskEvent(structs.TaskKilling).
@@ -565,23 +567,22 @@ func maskProcessEnv(env map[string]string) map[string]string {
 	return env
 }
 
+// checkEscape returns true if the absolute path testPath escapes both the
+// task working directory and the shared allocation directory.
+func (c *TaskTemplateManagerConfig) checkEscape(test string) bool {
+	for _, p := range []string{c.SharedAllocDir, c.TaskDir} {
+		if !helper.PathEscapesSandbox(p, test) {
+			return false
+		}
+	}
+	return true
+}
+
 // parseTemplateConfigs converts the tasks templates in the config into
 // consul-templates
 func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.TemplateConfig]*structs.Template, error) {
 	sandboxEnabled := !config.ClientConfig.TemplateConfig.DisableSandbox
-	taskEnv := config.EnvBuilder.Build()
-
-	// Make NOMAD_{ALLOC,TASK,SECRETS}_DIR relative paths to avoid treating
-	// them as sandbox escapes when using containers.
-	if taskEnv.EnvMap[taskenv.AllocDir] == allocdir.SharedAllocContainerPath {
-		taskEnv.EnvMap[taskenv.AllocDir] = allocdir.SharedAllocName
-	}
-	if taskEnv.EnvMap[taskenv.TaskLocalDir] == allocdir.TaskLocalContainerPath {
-		taskEnv.EnvMap[taskenv.TaskLocalDir] = allocdir.TaskLocal
-	}
-	if taskEnv.EnvMap[taskenv.SecretsDir] == allocdir.TaskSecretsContainerPath {
-		taskEnv.EnvMap[taskenv.SecretsDir] = allocdir.TaskSecrets
-	}
+	taskEnv := config.EnvBuilder.BuildClient()
 
 	ctmpls := make(map[*ctconf.TemplateConfig]*structs.Template, len(config.Templates))
 	for _, tmpl := range config.Templates {
@@ -590,22 +591,21 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa
 			src = taskEnv.ReplaceEnv(tmpl.SourcePath)
 			if !filepath.IsAbs(src) {
 				src = filepath.Join(config.TaskDir, src)
-			} else {
-				src = filepath.Clean(src)
 			}
-			escapes := helper.PathEscapesSandbox(config.TaskDir, src)
+			src = filepath.Clean(src)
+			escapes := config.checkEscape(src)
 			if escapes && sandboxEnabled {
 				return nil, sourceEscapesErr
 			}
 		}
 
 		if tmpl.DestPath != "" {
 			dest = taskEnv.ReplaceEnv(tmpl.DestPath)
-			// Note: we *always* join here even if we get passed an absolute
-			// path so that $NOMAD_SECRETS_DIR and friends can be used and
-			// always fall inside the task working directory
-			dest = filepath.Join(config.TaskDir, dest)
-			escapes := helper.PathEscapesSandbox(config.TaskDir, dest)
+			if !filepath.IsAbs(dest) {
+				dest = filepath.Join(config.TaskDir, dest)
+			}
+			dest = filepath.Clean(dest)
+			escapes := config.checkEscape(dest)
 			if escapes && sandboxEnabled {
 				return nil, destEscapesErr
 			}
@@ -740,14 +740,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 }
 
 // loadTemplateEnv loads task environment variables from all templates.
-func loadTemplateEnv(tmpls []*structs.Template, taskDir string, taskEnv *taskenv.TaskEnv) (map[string]string, error) {
+func loadTemplateEnv(tmpls []*structs.Template, taskDir string, envBuilder *taskenv.Builder) (map[string]string, error) {
+	clientEnv := envBuilder.BuildClient()
 	all := make(map[string]string, 50)
 	for _, t := range tmpls {
 		if !t.Envvars {
 			continue
 		}
 
-		dest := filepath.Join(taskDir, taskEnv.ReplaceEnv(t.DestPath))
+		dest := filepath.Join(taskDir, clientEnv.ReplaceEnv(t.DestPath))
 		f, err := os.Open(dest)
 		if err != nil {
 			return nil, fmt.Errorf("error opening env template: %v", err)

client/allocrunner/taskrunner/template_hook.go

@@ -52,6 +52,9 @@ type templateHook struct {
 
 	// taskDir is the task directory
 	taskDir string
+
+	// shared alloc dir is the shared task alloc dir
+	sharedAllocDir string
 }
 
 func newTemplateHook(config *templateHookConfig) *templateHook {
@@ -77,6 +80,7 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 
 	// Store the current Vault token and the task directory
 	h.taskDir = req.TaskDir.Dir
+	h.sharedAllocDir = req.TaskDir.SharedAllocDir
 	h.vaultToken = req.VaultToken
 
 	// Set vault namespace if specified
@@ -109,6 +113,7 @@ func (h *templateHook) newManager() (unblock chan struct{}, err error) {
 		VaultToken:           h.vaultToken,
 		VaultNamespace:       h.vaultNamespace,
 		TaskDir:              h.taskDir,
+		SharedAllocDir:       h.sharedAllocDir,
 		EnvBuilder:           h.config.envBuilder,
 		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
 	})