template: custom change_mode scripts

.changelog/13972.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+template: add script change_mode that allows scripts to be executed on template change
+```

api/tasks.go

@@ -791,21 +791,44 @@ func (wc *WaitConfig) Copy() *WaitConfig {
 	return nwc
 }
 
+type ChangeScriptConfig struct {
+	Path        *string        `mapstructure:"path" hcl:"path"`
+	Args        *[]string      `mapstructure:"args" hcl:"args,optional"`
+	Timeout     *time.Duration `mapstructure:"timeout" hcl:"timeout,optional"`
+	FailOnError *bool          `mapstructure:"fail_on_error" hcl:"fail_on_error"`
+}
+
+func (ch *ChangeScriptConfig) Canonicalize() {
+	if ch.Path == nil {
+		ch.Path = stringToPtr("")
+	}
+	if ch.Args == nil {
+		ch.Args = &[]string{}
+	}
+	if ch.Timeout == nil {
+		ch.Timeout = timeToPtr(5 * time.Second)
+	}
+	if ch.FailOnError == nil {
+		ch.FailOnError = boolToPtr(false)
+	}
+}
+
 type Template struct {
-	SourcePath   *string        `mapstructure:"source" hcl:"source,optional"`
-	DestPath     *string        `mapstructure:"destination" hcl:"destination,optional"`
-	EmbeddedTmpl *string        `mapstructure:"data" hcl:"data,optional"`
-	ChangeMode   *string        `mapstructure:"change_mode" hcl:"change_mode,optional"`
-	ChangeSignal *string        `mapstructure:"change_signal" hcl:"change_signal,optional"`
-	Splay        *time.Duration `mapstructure:"splay" hcl:"splay,optional"`
-	Perms        *string        `mapstructure:"perms" hcl:"perms,optional"`
-	Uid          *int           `mapstructure:"uid" hcl:"uid,optional"`
-	Gid          *int           `mapstructure:"gid" hcl:"gid,optional"`
-	LeftDelim    *string        `mapstructure:"left_delimiter" hcl:"left_delimiter,optional"`
-	RightDelim   *string        `mapstructure:"right_delimiter" hcl:"right_delimiter,optional"`
-	Envvars      *bool          `mapstructure:"env" hcl:"env,optional"`
-	VaultGrace   *time.Duration `mapstructure:"vault_grace" hcl:"vault_grace,optional"`
-	Wait         *WaitConfig    `mapstructure:"wait" hcl:"wait,block"`
+	SourcePath         *string             `mapstructure:"source" hcl:"source,optional"`
+	DestPath           *string             `mapstructure:"destination" hcl:"destination,optional"`
+	EmbeddedTmpl       *string             `mapstructure:"data" hcl:"data,optional"`
+	ChangeMode         *string             `mapstructure:"change_mode" hcl:"change_mode,optional"`
+	ChangeScriptConfig *ChangeScriptConfig `mapstructure:"change_script_config" hcl:"change_script_config,block"`
+	ChangeSignal       *string             `mapstructure:"change_signal" hcl:"change_signal,optional"`
+	Splay              *time.Duration      `mapstructure:"splay" hcl:"splay,optional"`
+	Perms              *string             `mapstructure:"perms" hcl:"perms,optional"`
+	Uid                *int                `mapstructure:"uid" hcl:"uid,optional"`
+	Gid                *int                `mapstructure:"gid" hcl:"gid,optional"`
+	LeftDelim          *string             `mapstructure:"left_delimiter" hcl:"left_delimiter,optional"`
+	RightDelim         *string             `mapstructure:"right_delimiter" hcl:"right_delimiter,optional"`
+	Envvars            *bool               `mapstructure:"env" hcl:"env,optional"`
+	VaultGrace         *time.Duration      `mapstructure:"vault_grace" hcl:"vault_grace,optional"`
+	Wait               *WaitConfig         `mapstructure:"wait" hcl:"wait,block"`
 }
 
 func (tmpl *Template) Canonicalize() {
@@ -831,6 +854,9 @@ func (tmpl *Template) Canonicalize() {
 		sig := *tmpl.ChangeSignal
 		tmpl.ChangeSignal = stringToPtr(strings.ToUpper(sig))
 	}
+	if tmpl.ChangeScriptConfig != nil {
+		tmpl.ChangeScriptConfig.Canonicalize()
+	}
 	if tmpl.Splay == nil {
 		tmpl.Splay = timeToPtr(5 * time.Second)
 	}

client/allocrunner/taskrunner/template/template.go

@@ -108,6 +108,9 @@ type TaskTemplateManagerConfig struct {
 
 	// NomadToken is the Nomad token or identity claim for the task
 	NomadToken string
+
+	// Handle is used to execute scripts
+	Handle interfaces.ScriptExecutor
 }
 
 // Validate validates the configuration.
@@ -392,6 +395,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 
 	var handling []string
 	signals := make(map[string]struct{})
+	scripts := []*structs.ChangeScriptConfig{}
 	restart := false
 	var splay time.Duration
 
@@ -436,6 +440,8 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 				signals[tmpl.ChangeSignal] = struct{}{}
 			case structs.TemplateChangeModeRestart:
 				restart = true
+			case structs.TemplateChangeModeScript:
+				scripts = append(scripts, tmpl.ChangeScriptConfig)
 			case structs.TemplateChangeModeNoop:
 				continue
 			}
@@ -493,6 +499,32 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 			}
 		}
 	}
+	if len(scripts) != 0 {
+		for _, script := range scripts {
+			_, exitCode, err := tm.config.Handle.Exec(script.Timeout, script.Path, script.Args)
+			if err != nil {
+				structs.NewTaskEvent(structs.TaskHookFailed).
+					SetDisplayMessage(
+						fmt.Sprintf(
+							"Template failed to run script %v on change: %v Exit code: %v", script.Path, err, exitCode,
+						))
+				if script.FailOnError {
+					tm.config.Lifecycle.Kill(context.Background(),
+						structs.NewTaskEvent(structs.TaskKilling).
+							SetFailsTask().
+							SetDisplayMessage(
+								fmt.Sprintf("Template failed to run script %v and the task is being killed", script.Path),
+							))
+				}
+			} else {
+				tm.config.Events.EmitEvent(structs.NewTaskEvent(structs.TaskHookMessage).
+					SetDisplayMessage(
+						fmt.Sprintf(
+							"Template successfully ran a script from %v with exit code: %v", script.Path, exitCode,
+						)))
+			}
+		}
+	}
 
 }
 

client/allocrunner/taskrunner/template/template_test.go

@@ -24,6 +24,7 @@ import (
 	ctestutil "github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/helper"
@@ -123,6 +124,16 @@ func (m *MockTaskHooks) EmitEvent(event *structs.TaskEvent) {
 
 func (m *MockTaskHooks) SetState(state string, event *structs.TaskEvent) {}
 
+// mockExecutor implements script executor interface
+type mockExecutor struct {
+	DesiredExit int
+	DesiredErr  error
+}
+
+func (m *mockExecutor) Exec(timeout time.Duration, cmd string, args []string) ([]byte, int, error) {
+	return []byte{}, m.DesiredExit, m.DesiredErr
+}
+
 // testHarness is used to test the TaskTemplateManager by spinning up
 // Consul/Vault as needed
 type testHarness struct {
@@ -138,6 +149,7 @@ type testHarness struct {
 	consul         *ctestutil.TestServer
 	emitRate       time.Duration
 	nomadNamespace string
+	driver         interfaces.ScriptExecutor
 }
 
 // newTestHarness returns a harness starting a dev consul and vault server,
@@ -211,6 +223,7 @@ func (h *testHarness) startWithErr() error {
 		VaultToken:           h.vaultToken,
 		TaskDir:              h.taskDir,
 		EnvBuilder:           h.envBuilder,
+		Handle:               h.driver,
 		MaxTemplateEventRate: h.emitRate,
 	})
 
@@ -1201,6 +1214,168 @@ func TestTaskTemplateManager_Signal_Error(t *testing.T) {
 	require.Contains(harness.mockHooks.KillEvent.DisplayMessage, "failed to send signals")
 }
 
+func TestTaskTemplateManager_ScriptExecution(t *testing.T) {
+	ci.Parallel(t)
+
+	// Make a template that renders based on a key in Consul and triggers script
+	key1 := "bam"
+	key2 := "bar"
+	content1_1 := "cat"
+	content1_2 := "dog"
+	t1 := &structs.Template{
+		EmbeddedTmpl: `
+FOO={{key "bam"}}
+`,
+		DestPath:   "test.env",
+		ChangeMode: structs.TemplateChangeModeScript,
+		ChangeScriptConfig: &structs.ChangeScriptConfig{
+			Path:        "/bin/foo",
+			Args:        []string{},
+			Timeout:     5 * time.Second,
+			FailOnError: false,
+		},
+		Envvars: true,
+	}
+	t2 := &structs.Template{
+		EmbeddedTmpl: `
+BAR={{key "bar"}}
+`,
+		DestPath:   "test2.env",
+		ChangeMode: structs.TemplateChangeModeScript,
+		ChangeScriptConfig: &structs.ChangeScriptConfig{
+			Path:        "/bin/foo",
+			Args:        []string{},
+			Timeout:     5 * time.Second,
+			FailOnError: false,
+		},
+		Envvars: true,
+	}
+
+	me := mockExecutor{}
+	harness := newTestHarness(t, []*structs.Template{t1, t2}, true, false)
+	harness.driver = &me
+	harness.start(t)
+	defer harness.stop()
+
+	// Ensure no unblock
+	select {
+	case <-harness.mockHooks.UnblockCh:
+		require.Fail(t, "Task unblock should not have been called")
+	case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
+	}
+
+	// Write the key to Consul
+	harness.consul.SetKV(t, key1, []byte(content1_1))
+	harness.consul.SetKV(t, key2, []byte(content1_1))
+
+	// Wait for the unblock
+	select {
+	case <-harness.mockHooks.UnblockCh:
+	case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
+		require.Fail(t, "Task unblock should have been called")
+	}
+
+	// Update the keys in Consul
+	harness.consul.SetKV(t, key1, []byte(content1_2))
+
+	// Wait for restart
+	timeout := time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second)
+OUTER:
+	for {
+		select {
+		case <-harness.mockHooks.RestartCh:
+			require.Fail(t, "restart not expected")
+		case ev := <-harness.mockHooks.EmitEventCh:
+			if strings.Contains(ev.DisplayMessage, t1.ChangeScriptConfig.Path) {
+				break OUTER
+			}
+		case <-harness.mockHooks.SignalCh:
+			require.Fail(t, "signal not expected")
+		case <-timeout:
+			require.Fail(t, "should have received an event")
+		}
+	}
+}
+
+// TestTaskTemplateManager_ScriptExecutionFailTask tests whether we fail the
+// task upon script execution failure if that's how it's configured.
+func TestTaskTemplateManager_ScriptExecutionFailTask(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	// Make a template that renders based on a key in Consul and triggers script
+	key1 := "bam"
+	key2 := "bar"
+	content1_1 := "cat"
+	content1_2 := "dog"
+	t1 := &structs.Template{
+		EmbeddedTmpl: `
+FOO={{key "bam"}}
+`,
+		DestPath:   "test.env",
+		ChangeMode: structs.TemplateChangeModeScript,
+		ChangeScriptConfig: &structs.ChangeScriptConfig{
+			Path:        "/bin/foo",
+			Args:        []string{},
+			Timeout:     5 * time.Second,
+			FailOnError: true,
+		},
+		Envvars: true,
+	}
+	t2 := &structs.Template{
+		EmbeddedTmpl: `
+BAR={{key "bar"}}
+`,
+		DestPath:   "test2.env",
+		ChangeMode: structs.TemplateChangeModeScript,
+		ChangeScriptConfig: &structs.ChangeScriptConfig{
+			Path:        "/bin/foo",
+			Args:        []string{},
+			Timeout:     5 * time.Second,
+			FailOnError: false,
+		},
+		Envvars: true,
+	}
+
+	me := mockExecutor{DesiredExit: 1, DesiredErr: fmt.Errorf("Script failed")}
+	harness := newTestHarness(t, []*structs.Template{t1, t2}, true, false)
+	harness.driver = &me
+	harness.start(t)
+	defer harness.stop()
+
+	// Ensure no unblock
+	select {
+	case <-harness.mockHooks.UnblockCh:
+		require.Fail("Task unblock should not have been called")
+	case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
+	}
+
+	// Write the key to Consul
+	harness.consul.SetKV(t, key1, []byte(content1_1))
+	harness.consul.SetKV(t, key2, []byte(content1_1))
+
+	// Wait for the unblock
+	select {
+	case <-harness.mockHooks.UnblockCh:
+	case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
+		require.Fail("Task unblock should have been called")
+	}
+
+	// Update the keys in Consul
+	harness.consul.SetKV(t, key1, []byte(content1_2))
+
+	// Wait for kill channel
+	select {
+	case <-harness.mockHooks.KillCh:
+		break
+	case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
+		require.Fail("Should have received a signals: %+v", harness.mockHooks)
+	}
+
+	require.NotNil(harness.mockHooks.KillEvent)
+	require.Contains(harness.mockHooks.KillEvent.DisplayMessage, "failed to run script")
+}
+
 // TestTaskTemplateManager_FiltersProcessEnvVars asserts that we only render
 // environment variables found in task env-vars and not read the nomad host
 // process environment variables.  nomad host process environment variables

client/allocrunner/taskrunner/template_hook.go

@@ -42,6 +42,9 @@ type templateHookConfig struct {
 
 	// nomadNamespace is the job's Nomad namespace
 	nomadNamespace string
+
+	// handle is the driver handle that allows driver operations
+	handle *DriverHandle
 }
 
 type templateHook struct {
@@ -131,6 +134,7 @@ func (h *templateHook) newManager() (unblock chan struct{}, err error) {
 		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
 		NomadNamespace:       h.config.nomadNamespace,
 		NomadToken:           h.nomadToken,
+		Handle:               h.config.handle,
 	})
 	if err != nil {
 		h.logger.Error("failed to create template manager", "error", err)