template: error on missing key

.changelog/14002.txt

@@ -0,0 +1,5 @@
+```release-note:improvement
+template: Expose per-template configuration for `error_on_missing_key`. This allows jobspec authors to specify that a
+template should fail if it references a struct or map key that does not exist. The default value is false and should be
+fully backward compatible.
+```

api/jobs_test.go

@@ -758,34 +758,36 @@ func TestJobs_Canonicalize(t *testing.T) {
 								LogConfig:   DefaultLogConfig(),
 								Templates: []*Template{
 									{
-										SourcePath:   stringToPtr(""),
-										DestPath:     stringToPtr("local/file.yml"),
-										EmbeddedTmpl: stringToPtr("---"),
-										ChangeMode:   stringToPtr("restart"),
-										ChangeSignal: stringToPtr(""),
-										Splay:        timeToPtr(5 * time.Second),
-										Perms:        stringToPtr("0644"),
-										Uid:          intToPtr(-1),
-										Gid:          intToPtr(-1),
-										LeftDelim:    stringToPtr("{{"),
-										RightDelim:   stringToPtr("}}"),
-										Envvars:      boolToPtr(false),
-										VaultGrace:   timeToPtr(0),
+										SourcePath:    stringToPtr(""),
+										DestPath:      stringToPtr("local/file.yml"),
+										EmbeddedTmpl:  stringToPtr("---"),
+										ChangeMode:    stringToPtr("restart"),
+										ChangeSignal:  stringToPtr(""),
+										Splay:         timeToPtr(5 * time.Second),
+										Perms:         stringToPtr("0644"),
+										Uid:           intToPtr(-1),
+										Gid:           intToPtr(-1),
+										LeftDelim:     stringToPtr("{{"),
+										RightDelim:    stringToPtr("}}"),
+										Envvars:       boolToPtr(false),
+										VaultGrace:    timeToPtr(0),
+										ErrMissingKey: boolToPtr(false),
 									},
 									{
-										SourcePath:   stringToPtr(""),
-										DestPath:     stringToPtr("local/file.env"),
-										EmbeddedTmpl: stringToPtr("FOO=bar\n"),
-										ChangeMode:   stringToPtr("restart"),
-										ChangeSignal: stringToPtr(""),
-										Splay:        timeToPtr(5 * time.Second),
-										Perms:        stringToPtr("0644"),
-										Uid:          intToPtr(-1),
-										Gid:          intToPtr(-1),
-										LeftDelim:    stringToPtr("{{"),
-										RightDelim:   stringToPtr("}}"),
-										Envvars:      boolToPtr(true),
-										VaultGrace:   timeToPtr(0),
+										SourcePath:    stringToPtr(""),
+										DestPath:      stringToPtr("local/file.env"),
+										EmbeddedTmpl:  stringToPtr("FOO=bar\n"),
+										ChangeMode:    stringToPtr("restart"),
+										ChangeSignal:  stringToPtr(""),
+										Splay:         timeToPtr(5 * time.Second),
+										Perms:         stringToPtr("0644"),
+										Uid:           intToPtr(-1),
+										Gid:           intToPtr(-1),
+										LeftDelim:     stringToPtr("{{"),
+										RightDelim:    stringToPtr("}}"),
+										Envvars:       boolToPtr(true),
+										VaultGrace:    timeToPtr(0),
+										ErrMissingKey: boolToPtr(false),
 									},
 								},
 							},

api/tasks.go

@@ -792,20 +792,21 @@ func (wc *WaitConfig) Copy() *WaitConfig {
 }
 
 type Template struct {
-	SourcePath   *string        `mapstructure:"source" hcl:"source,optional"`
-	DestPath     *string        `mapstructure:"destination" hcl:"destination,optional"`
-	EmbeddedTmpl *string        `mapstructure:"data" hcl:"data,optional"`
-	ChangeMode   *string        `mapstructure:"change_mode" hcl:"change_mode,optional"`
-	ChangeSignal *string        `mapstructure:"change_signal" hcl:"change_signal,optional"`
-	Splay        *time.Duration `mapstructure:"splay" hcl:"splay,optional"`
-	Perms        *string        `mapstructure:"perms" hcl:"perms,optional"`
-	Uid          *int           `mapstructure:"uid" hcl:"uid,optional"`
-	Gid          *int           `mapstructure:"gid" hcl:"gid,optional"`
-	LeftDelim    *string        `mapstructure:"left_delimiter" hcl:"left_delimiter,optional"`
-	RightDelim   *string        `mapstructure:"right_delimiter" hcl:"right_delimiter,optional"`
-	Envvars      *bool          `mapstructure:"env" hcl:"env,optional"`
-	VaultGrace   *time.Duration `mapstructure:"vault_grace" hcl:"vault_grace,optional"`
-	Wait         *WaitConfig    `mapstructure:"wait" hcl:"wait,block"`
+	SourcePath    *string        `mapstructure:"source" hcl:"source,optional"`
+	DestPath      *string        `mapstructure:"destination" hcl:"destination,optional"`
+	EmbeddedTmpl  *string        `mapstructure:"data" hcl:"data,optional"`
+	ChangeMode    *string        `mapstructure:"change_mode" hcl:"change_mode,optional"`
+	ChangeSignal  *string        `mapstructure:"change_signal" hcl:"change_signal,optional"`
+	Splay         *time.Duration `mapstructure:"splay" hcl:"splay,optional"`
+	Perms         *string        `mapstructure:"perms" hcl:"perms,optional"`
+	Uid           *int           `mapstructure:"uid" hcl:"uid,optional"`
+	Gid           *int           `mapstructure:"gid" hcl:"gid,optional"`
+	LeftDelim     *string        `mapstructure:"left_delimiter" hcl:"left_delimiter,optional"`
+	RightDelim    *string        `mapstructure:"right_delimiter" hcl:"right_delimiter,optional"`
+	Envvars       *bool          `mapstructure:"env" hcl:"env,optional"`
+	VaultGrace    *time.Duration `mapstructure:"vault_grace" hcl:"vault_grace,optional"`
+	Wait          *WaitConfig    `mapstructure:"wait" hcl:"wait,block"`
+	ErrMissingKey *bool          `mapstructure:"error_on_missing_key" hcl:"error_on_missing_key,optional"`
 }
 
 func (tmpl *Template) Canonicalize() {
@@ -852,6 +853,9 @@ func (tmpl *Template) Canonicalize() {
 	if tmpl.Envvars == nil {
 		tmpl.Envvars = boolToPtr(false)
 	}
+	if tmpl.ErrMissingKey == nil {
+		tmpl.ErrMissingKey = boolToPtr(false)
+	}
 
 	//COMPAT(0.12) VaultGrace is deprecated and unused as of Vault 0.5
 	if tmpl.VaultGrace == nil {

client/allocrunner/taskrunner/template/template.go

@@ -600,6 +600,7 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa
 		ct.Contents = &tmpl.EmbeddedTmpl
 		ct.LeftDelim = &tmpl.LeftDelim
 		ct.RightDelim = &tmpl.RightDelim
+		ct.ErrMissingKey = &tmpl.ErrMissingKey
 		ct.FunctionDenylist = config.ClientConfig.TemplateConfig.FunctionDenylist
 		if sandboxEnabled {
 			ct.SandboxPath = &config.TaskDir

client/allocrunner/taskrunner/template/template_test.go

@@ -2176,6 +2176,46 @@ func TestTaskTemplateManager_Template_Wait_Set(t *testing.T) {
 	}
 }
 
+// TestTaskTemplateManager_Template_ErrMissingKey_Set asserts that all template level
+// configuration is accurately mapped from the template to the TaskTemplateManager's
+// template config.
+func TestTaskTemplateManager_Template_ErrMissingKey_Set(t *testing.T) {
+	ci.Parallel(t)
+
+	c := config.DefaultConfig()
+	c.Node = mock.Node()
+
+	alloc := mock.Alloc()
+
+	ttmConfig := &TaskTemplateManagerConfig{
+		ClientConfig: c,
+		VaultToken:   "token",
+		EnvBuilder:   taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region),
+		Templates: []*structs.Template{
+			{
+				EmbeddedTmpl:  "test-false",
+				ErrMissingKey: false,
+			},
+			{
+				EmbeddedTmpl:  "test-true",
+				ErrMissingKey: true,
+			},
+		},
+	}
+
+	templateMapping, err := parseTemplateConfigs(ttmConfig)
+	require.NoError(t, err)
+
+	for k, tmpl := range templateMapping {
+		if tmpl.EmbeddedTmpl == "test-false" {
+			require.False(t, *k.ErrMissingKey)
+		}
+		if tmpl.EmbeddedTmpl == "test-true" {
+			require.True(t, *k.ErrMissingKey)
+		}
+	}
+}
+
 // TestTaskTemplateManager_writeToFile_Disabled asserts the consul-template function
 // writeToFile is disabled by default.
 func TestTaskTemplateManager_writeToFile_Disabled(t *testing.T) {

command/agent/job_endpoint.go

@@ -1219,20 +1219,21 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
 			}
 			structsTask.Templates = append(structsTask.Templates,
 				&structs.Template{
-					SourcePath:   *template.SourcePath,
-					DestPath:     *template.DestPath,
-					EmbeddedTmpl: *template.EmbeddedTmpl,
-					ChangeMode:   *template.ChangeMode,
-					ChangeSignal: *template.ChangeSignal,
-					Splay:        *template.Splay,
-					Perms:        *template.Perms,
-					Uid:          uid,
-					Gid:          gid,
-					LeftDelim:    *template.LeftDelim,
-					RightDelim:   *template.RightDelim,
-					Envvars:      *template.Envvars,
-					VaultGrace:   *template.VaultGrace,
-					Wait:         ApiWaitConfigToStructsWaitConfig(template.Wait),
+					SourcePath:    *template.SourcePath,
+					DestPath:      *template.DestPath,
+					EmbeddedTmpl:  *template.EmbeddedTmpl,
+					ChangeMode:    *template.ChangeMode,
+					ChangeSignal:  *template.ChangeSignal,
+					Splay:         *template.Splay,
+					Perms:         *template.Perms,
+					Uid:           uid,
+					Gid:           gid,
+					LeftDelim:     *template.LeftDelim,
+					RightDelim:    *template.RightDelim,
+					Envvars:       *template.Envvars,
+					VaultGrace:    *template.VaultGrace,
+					Wait:          ApiWaitConfigToStructsWaitConfig(template.Wait),
+					ErrMissingKey: *template.ErrMissingKey,
 				})
 		}
 	}

command/agent/job_endpoint_test.go

@@ -14,6 +14,7 @@ import (
 	api "github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/stretchr/testify/assert"
@@ -2742,6 +2743,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 									Min: helper.TimeToPtr(5 * time.Second),
 									Max: helper.TimeToPtr(10 * time.Second),
 								},
+								ErrMissingKey: pointer.Of[bool](true),
 							},
 						},
 						DispatchPayload: &api.DispatchPayloadConfig{
@@ -3149,6 +3151,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 									Min: helper.TimeToPtr(5 * time.Second),
 									Max: helper.TimeToPtr(10 * time.Second),
 								},
+								ErrMissingKey: true,
 							},
 						},
 						DispatchPayload: &structs.DispatchPayloadConfig{

jobspec/parse_task.go

@@ -448,6 +448,8 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
 			"splay",
 			"env",
 			"vault_grace", //COMPAT(0.12) not used; emits warning in 0.11.
+			"wait",
+			"error_on_missing_key",
 		}
 		if err := checkHCLKeys(o.Val, valid); err != nil {
 			return err
@@ -459,11 +461,12 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
 		}
 
 		templ := &api.Template{
-			ChangeMode: stringToPtr("restart"),
-			Splay:      timeToPtr(5 * time.Second),
-			Perms:      stringToPtr("0644"),
-			Uid:        intToPtr(-1),
-			Gid:        intToPtr(-1),
+			ChangeMode:    stringToPtr("restart"),
+			Splay:         timeToPtr(5 * time.Second),
+			Perms:         stringToPtr("0644"),
+			Uid:           intToPtr(-1),
+			Gid:           intToPtr(-1),
+			ErrMissingKey: boolToPtr(false),
 		}
 
 		dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{

jobspec/parse_test.go

@@ -368,27 +368,29 @@ func TestParse(t *testing.T) {
 								},
 								Templates: []*api.Template{
 									{
-										SourcePath:   stringToPtr("foo"),
-										DestPath:     stringToPtr("foo"),
-										ChangeMode:   stringToPtr("foo"),
-										ChangeSignal: stringToPtr("foo"),
-										Splay:        timeToPtr(10 * time.Second),
-										Perms:        stringToPtr("0644"),
-										Uid:          intToPtr(-1),
-										Gid:          intToPtr(-1),
-										Envvars:      boolToPtr(true),
-										VaultGrace:   timeToPtr(33 * time.Second),
+										SourcePath:    stringToPtr("foo"),
+										DestPath:      stringToPtr("foo"),
+										ChangeMode:    stringToPtr("foo"),
+										ChangeSignal:  stringToPtr("foo"),
+										Splay:         timeToPtr(10 * time.Second),
+										Perms:         stringToPtr("0644"),
+										Uid:           intToPtr(-1),
+										Gid:           intToPtr(-1),
+										Envvars:       boolToPtr(true),
+										VaultGrace:    timeToPtr(33 * time.Second),
+										ErrMissingKey: boolToPtr(true),
 									},
 									{
-										SourcePath: stringToPtr("bar"),
-										DestPath:   stringToPtr("bar"),
-										ChangeMode: stringToPtr(templateChangeModeRestart),
-										Splay:      timeToPtr(5 * time.Second),
-										Perms:      stringToPtr("777"),
-										Uid:        intToPtr(1001),
-										Gid:        intToPtr(20),
-										LeftDelim:  stringToPtr("--"),
-										RightDelim: stringToPtr("__"),
+										SourcePath:    stringToPtr("bar"),
+										DestPath:      stringToPtr("bar"),
+										ChangeMode:    stringToPtr(templateChangeModeRestart),
+										Splay:         timeToPtr(5 * time.Second),
+										Perms:         stringToPtr("777"),
+										Uid:           intToPtr(1001),
+										Gid:           intToPtr(20),
+										LeftDelim:     stringToPtr("--"),
+										RightDelim:    stringToPtr("__"),
+										ErrMissingKey: boolToPtr(false),
 									},
 								},
 								Leader:     true,

jobspec/test-fixtures/basic.hcl

@@ -305,13 +305,14 @@ job "binstore-storagelocker" {
       }
 
       template {
-        source        = "foo"
-        destination   = "foo"
-        change_mode   = "foo"
-        change_signal = "foo"
-        splay         = "10s"
-        env           = true
-        vault_grace   = "33s"
+        source               = "foo"
+        destination          = "foo"
+        change_mode          = "foo"
+        change_signal        = "foo"
+        splay                = "10s"
+        env                  = true
+        vault_grace          = "33s"
+        error_on_missing_key = true
       }
 
       template {

jobspec2/parse_job.go

@@ -116,6 +116,9 @@ func normalizeTemplates(templates []*api.Template) {
 		if t.Splay == nil {
 			t.Splay = durationToPtr(5 * time.Second)
 		}
+		if t.ErrMissingKey == nil {
+			t.ErrMissingKey = boolToPtr(false)
+		}
 	}
 }
 

jobspec2/parse_test.go

@@ -1033,7 +1033,7 @@ func TestParseServiceCheck(t *testing.T) {
 
 func TestWaitConfig(t *testing.T) {
 	ci.Parallel(t)
-	
+
 	hclBytes, err := os.ReadFile("test-fixtures/template-wait-config.hcl")
 	require.NoError(t, err)
 
@@ -1051,3 +1051,23 @@ func TestWaitConfig(t *testing.T) {
 	require.Equal(t, 5*time.Second, *tmpl.Wait.Min)
 	require.Equal(t, 60*time.Second, *tmpl.Wait.Max)
 }
+
+func TestErrMissingKey(t *testing.T) {
+	ci.Parallel(t)
+
+	hclBytes, err := os.ReadFile("test-fixtures/template-err-missing-key.hcl")
+	require.NoError(t, err)
+
+	job, err := ParseWithConfig(&ParseConfig{
+		Path:    "test-fixtures/template-err-missing-key.hcl",
+		Body:    hclBytes,
+		AllowFS: false,
+	})
+
+	require.NoError(t, err)
+
+	tmpl := job.TaskGroups[0].Tasks[0].Templates[0]
+	require.NotNil(t, tmpl)
+	require.NotNil(t, tmpl.ErrMissingKey)
+	require.True(t, *tmpl.ErrMissingKey)
+}

jobspec2/test-fixtures/template-err-missing-key.hcl

@@ -0,0 +1,9 @@
+job "example" {
+  group "group" {
+    task "task" {
+      template {
+        error_on_missing_key = true
+      }
+    }
+  }
+}