template: custom change_mode scripts (#13972)

This PR adds the functionality of allowing custom scripts to be executed on template change. Resolves #2707

.changelog/13972.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+template: add script change_mode that allows scripts to be executed on template change
+```

api/tasks.go

@@ -809,11 +809,34 @@ func (wc *WaitConfig) Copy() *WaitConfig {
 	return nwc
 }
 
+type ChangeScript struct {
+	Command     *string        `mapstructure:"command" hcl:"command"`
+	Args        []string       `mapstructure:"args" hcl:"args,optional"`
+	Timeout     *time.Duration `mapstructure:"timeout" hcl:"timeout,optional"`
+	FailOnError *bool          `mapstructure:"fail_on_error" hcl:"fail_on_error"`
+}
+
+func (ch *ChangeScript) Canonicalize() {
+	if ch.Command == nil {
+		ch.Command = pointerOf("")
+	}
+	if ch.Args == nil {
+		ch.Args = []string{}
+	}
+	if ch.Timeout == nil {
+		ch.Timeout = pointerOf(5 * time.Second)
+	}
+	if ch.FailOnError == nil {
+		ch.FailOnError = pointerOf(false)
+	}
+}
+
 type Template struct {
 	SourcePath   *string        `mapstructure:"source" hcl:"source,optional"`
 	DestPath     *string        `mapstructure:"destination" hcl:"destination,optional"`
 	EmbeddedTmpl *string        `mapstructure:"data" hcl:"data,optional"`
 	ChangeMode   *string        `mapstructure:"change_mode" hcl:"change_mode,optional"`
+	ChangeScript *ChangeScript  `mapstructure:"change_script" hcl:"change_script,block"`
 	ChangeSignal *string        `mapstructure:"change_signal" hcl:"change_signal,optional"`
 	Splay        *time.Duration `mapstructure:"splay" hcl:"splay,optional"`
 	Perms        *string        `mapstructure:"perms" hcl:"perms,optional"`
@@ -849,6 +872,9 @@ func (tmpl *Template) Canonicalize() {
 		sig := *tmpl.ChangeSignal
 		tmpl.ChangeSignal = pointerOf(strings.ToUpper(sig))
 	}
+	if tmpl.ChangeScript != nil {
+		tmpl.ChangeScript.Canonicalize()
+	}
 	if tmpl.Splay == nil {
 		tmpl.Splay = pointerOf(5 * time.Second)
 	}

client/allocrunner/taskrunner/template/template.go

@@ -54,6 +54,10 @@ type TaskTemplateManager struct {
 	// runner is the consul-template runner
 	runner *manager.Runner
 
+	// handle is used to execute scripts
+	handle     interfaces.ScriptExecutor
+	handleLock sync.Mutex
+
 	// signals is a lookup map from the string representation of a signal to its
 	// actual signal
 	signals map[string]os.Signal
@@ -192,6 +196,14 @@ func (tm *TaskTemplateManager) Stop() {
 	}
 }
 
+// SetDriverHandle sets the executor
+func (tm *TaskTemplateManager) SetDriverHandle(executor interfaces.ScriptExecutor) {
+	tm.handleLock.Lock()
+	defer tm.handleLock.Unlock()
+	tm.handle = executor
+
+}
+
 // run is the long lived loop that handles errors and templates being rendered
 func (tm *TaskTemplateManager) run() {
 	// Runner is nil if there are no templates
@@ -392,6 +404,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 
 	var handling []string
 	signals := make(map[string]struct{})
+	scripts := []*structs.ChangeScript{}
 	restart := false
 	var splay time.Duration
 
@@ -436,6 +449,8 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 				signals[tmpl.ChangeSignal] = struct{}{}
 			case structs.TemplateChangeModeRestart:
 				restart = true
+			case structs.TemplateChangeModeScript:
+				scripts = append(scripts, tmpl.ChangeScript)
 			case structs.TemplateChangeModeNoop:
 				continue
 			}
@@ -494,6 +509,72 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
 		}
 	}
 
+	// process script execution concurrently
+	var wg sync.WaitGroup
+	for _, script := range scripts {
+		wg.Add(1)
+		go tm.processScript(script, &wg)
+	}
+	wg.Wait()
+}
+
+// handleScriptError is a helper function that produces a TaskKilling event and
+// emits a message
+func (tm *TaskTemplateManager) handleScriptError(script *structs.ChangeScript, msg string) {
+	ev := structs.NewTaskEvent(structs.TaskHookFailed).SetDisplayMessage(msg)
+	tm.config.Events.EmitEvent(ev)
+
+	if script.FailOnError {
+		tm.config.Lifecycle.Kill(context.Background(),
+			structs.NewTaskEvent(structs.TaskKilling).
+				SetFailsTask().
+				SetDisplayMessage("Template script failed, task is being killed"))
+	}
+}
+
+// processScript is used for executing change_mode script and handling errors
+func (tm *TaskTemplateManager) processScript(script *structs.ChangeScript, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	if tm.handle == nil {
+		failureMsg := fmt.Sprintf(
+			"Template failed to run script %v with arguments %v because task driver doesn't support the exec operation",
+			script.Command,
+			script.Args,
+		)
+		tm.handleScriptError(script, failureMsg)
+		return
+	}
+	_, exitCode, err := tm.handle.Exec(script.Timeout, script.Command, script.Args)
+	if err != nil {
+		failureMsg := fmt.Sprintf(
+			"Template failed to run script %v with arguments %v on change: %v Exit code: %v",
+			script.Command,
+			script.Args,
+			err,
+			exitCode,
+		)
+		tm.handleScriptError(script, failureMsg)
+		return
+	}
+	if exitCode != 0 {
+		failureMsg := fmt.Sprintf(
+			"Template ran script %v with arguments %v on change but it exited with code code: %v",
+			script.Command,
+			script.Args,
+			exitCode,
+		)
+		tm.handleScriptError(script, failureMsg)
+		return
+	}
+	tm.config.Events.EmitEvent(structs.NewTaskEvent(structs.TaskHookMessage).
+		SetDisplayMessage(
+			fmt.Sprintf(
+				"Template successfully ran script %v with arguments: %v. Exit code: %v",
+				script.Command,
+				script.Args,
+				exitCode,
+			)))
 }
 
 // allTemplatesNoop returns whether all the managed templates have change mode noop.