Expose Consul template configuration parameters (#11606)

This PR exposes the following existing`consul-template` configuration options to Nomad jobspec authors in the `{job.group.task.template}` stanza. - `wait` It also exposes the following`consul-template` configuration to Nomad operators in the `{client.template}` stanza. - `max_stale` - `block_query_wait` - `consul_retry` - `vault_retry` - `wait` Finally, it adds the following new Nomad-specific configuration to the `{client.template}` stanza that allows Operators to set bounds on what `jobspec` authors configure. - `wait_bounds` Co-authored-by: Tim Gross <tgross@hashicorp.com> Co-authored-by: Michael Schurter <mschurter@hashicorp.com>
hashicorp · Jan 10, 2022 · 43edd0e · 43edd0e
1 parent 04095bd
commit 43edd0e
Show file tree

Hide file tree

Showing 24 changed files with 2,271 additions and 91 deletions.
diff --git a/.changelog/11606.txt b/.changelog/11606.txt
@@ -0,0 +1,6 @@
+```release-note:improvement
+template: Expose consul-template configuration options at the client level for `consul_retry`,
+`vault_retry`, `max_stale`, `block_query_wait` and `wait`. Expose per-template configuration
+for wait that will override the client level configuration. Add `wait_bounds` to
+allow operators to constrain per-template overrides at the client level.
+```
diff --git a/api/tasks.go b/api/tasks.go
@@ -772,6 +772,24 @@ func (a *TaskArtifact) Canonicalize() {
 	}
 }
 
+// WaitConfig is the Min/Max duration to wait for the Consul cluster to reach a
+// consistent state before attempting to render Templates.
+type WaitConfig struct {
+	Min *time.Duration `mapstructure:"min" hcl:"min"`
+	Max *time.Duration `mapstructure:"max" hcl:"max"`
+}
+
+func (wc *WaitConfig) Copy() *WaitConfig {
+	if wc == nil {
+		return nil
+	}
+
+	nwc := new(WaitConfig)
+	*nwc = *wc
+
+	return nwc
+}
+
 type Template struct {
 	SourcePath   *string        `mapstructure:"source" hcl:"source,optional"`
 	DestPath     *string        `mapstructure:"destination" hcl:"destination,optional"`
@@ -784,6 +802,7 @@ type Template struct {
 	RightDelim   *string        `mapstructure:"right_delimiter" hcl:"right_delimiter,optional"`
 	Envvars      *bool          `mapstructure:"env" hcl:"env,optional"`
 	VaultGrace   *time.Duration `mapstructure:"vault_grace" hcl:"vault_grace,optional"`
+	Wait         *WaitConfig    `mapstructure:"wait" hcl:"wait,block"`
 }
 
 func (tmpl *Template) Canonicalize() {

diff --git a/api/tasks_test.go b/api/tasks_test.go
@@ -406,6 +406,91 @@ func TestTask_Canonicalize_TaskLifecycle(t *testing.T) {
 	}
 }
 
+func TestTask_Template_WaitConfig_Canonicalize_and_Copy(t *testing.T) {
+	taskWithWait := func(wc *WaitConfig) *Task {
+		return &Task{
+			Templates: []*Template{
+				{
+					Wait: wc,
+				},
+			},
+		}
+	}
+
+	testCases := []struct {
+		name          string
+		canonicalized *WaitConfig
+		copied        *WaitConfig
+		task          *Task
+	}{
+		{
+			name: "all-fields",
+			task: taskWithWait(&WaitConfig{
+				Min: timeToPtr(5),
+				Max: timeToPtr(10),
+			}),
+			canonicalized: &WaitConfig{
+				Min: timeToPtr(5),
+				Max: timeToPtr(10),
+			},
+			copied: &WaitConfig{
+				Min: timeToPtr(5),
+				Max: timeToPtr(10),
+			},
+		},
+		{
+			name: "no-fields",
+			task: taskWithWait(&WaitConfig{}),
+			canonicalized: &WaitConfig{
+				Min: nil,
+				Max: nil,
+			},
+			copied: &WaitConfig{
+				Min: nil,
+				Max: nil,
+			},
+		},
+		{
+			name: "min-only",
+			task: taskWithWait(&WaitConfig{
+				Min: timeToPtr(5),
+			}),
+			canonicalized: &WaitConfig{
+				Min: timeToPtr(5),
+			},
+			copied: &WaitConfig{
+				Min: timeToPtr(5),
+			},
+		},
+		{
+			name: "max-only",
+			task: taskWithWait(&WaitConfig{
+				Max: timeToPtr(10),
+			}),
+			canonicalized: &WaitConfig{
+				Max: timeToPtr(10),
+			},
+			copied: &WaitConfig{
+				Max: timeToPtr(10),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tg := &TaskGroup{
+				Name: stringToPtr("foo"),
+			}
+			j := &Job{
+				ID: stringToPtr("test"),
+			}
+			require.Equal(t, tc.copied, tc.task.Templates[0].Wait.Copy())
+			tc.task.Canonicalize(tg, j)
+			require.Equal(t, tc.canonicalized, tc.task.Templates[0].Wait)
+		})
+	}
+}
+
 // Ensures no regression on https://github.com/hashicorp/nomad/issues/3132
 func TestTaskGroup_Canonicalize_Update(t *testing.T) {
 	// Job with an Empty() Update

diff --git a/client/allocrunner/taskrunner/template/template.go b/client/allocrunner/taskrunner/template/template.go
@@ -102,9 +102,6 @@ type TaskTemplateManagerConfig struct {
 
 	// MaxTemplateEventRate is the maximum rate at which we should emit events.
 	MaxTemplateEventRate time.Duration
-
-	// retryRate is only used for testing and is used to increase the retry rate
-	retryRate time.Duration
 }
 
 // Validate validates the configuration.
@@ -191,7 +188,7 @@ func (tm *TaskTemplateManager) Stop() {
 
 // run is the long lived loop that handles errors and templates being rendered
 func (tm *TaskTemplateManager) run() {
-	// Runner is nil if there is no templates
+	// Runner is nil if there are no templates
 	if tm.runner == nil {
 		// Unblock the start if there is nothing to do
 		close(tm.config.UnblockCh)
@@ -602,6 +599,18 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa
 			ct.SandboxPath = &config.TaskDir
 		}
 
+		if tmpl.Wait != nil {
+			if err := tmpl.Wait.Validate(); err != nil {
+				return nil, err
+			}
+
+			ct.Wait = &ctconf.WaitConfig{
+				Enabled: helper.BoolToPtr(true),
+				Min:     tmpl.Wait.Min,
+				Max:     tmpl.Wait.Max,
+			}
+		}
+
 		// Set the permissions
 		if tmpl.Perms != "" {
 			v, err := strconv.ParseUint(tmpl.Perms, 8, 12)
@@ -635,13 +644,60 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 	}
 	conf.Templates = &flat
 
-	// Force faster retries
-	if config.retryRate != 0 {
-		rate := config.retryRate
-		conf.Consul.Retry.Backoff = &rate
+	// Set the amount of time to do a blocking query for.
+	if cc.TemplateConfig.BlockQueryWaitTime != nil {
+		conf.BlockQueryWaitTime = cc.TemplateConfig.BlockQueryWaitTime
+	}
+
+	// Set the stale-read threshold to allow queries to be served by followers
+	// if the last replicated data is within this bound.
+	if cc.TemplateConfig.MaxStale != nil {
+		conf.MaxStale = cc.TemplateConfig.MaxStale
+	}
+
+	// Set the minimum and maximum amount of time to wait for the cluster to reach
+	// a consistent state before rendering a template.
+	if cc.TemplateConfig.Wait != nil {
+		// If somehow the WaitConfig wasn't set correctly upstream, return an error.
+		var err error
+		err = cc.TemplateConfig.Wait.Validate()
+		if err != nil {
+			return nil, err
+		}
+		conf.Wait, err = cc.TemplateConfig.Wait.ToConsulTemplate()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Make sure any template specific configuration set by the job author is within
+	// the bounds set by the operator.
+	if cc.TemplateConfig.WaitBounds != nil {
+		// If somehow the WaitBounds weren't set correctly upstream, return an error.
+		err := cc.TemplateConfig.WaitBounds.Validate()
+		if err != nil {
+			return nil, err
+		}
+
+		// Check and override with bounds
+		for _, tmpl := range *conf.Templates {
+			if tmpl.Wait == nil || !*tmpl.Wait.Enabled {
+				continue
+			}
+			if cc.TemplateConfig.WaitBounds.Min != nil {
+				if tmpl.Wait.Min != nil && *tmpl.Wait.Min < *cc.TemplateConfig.WaitBounds.Min {
+					tmpl.Wait.Min = &*cc.TemplateConfig.WaitBounds.Min
+				}
+			}
+			if cc.TemplateConfig.WaitBounds.Max != nil {
+				if tmpl.Wait.Max != nil && *tmpl.Wait.Max > *cc.TemplateConfig.WaitBounds.Max {
+					tmpl.Wait.Max = &*cc.TemplateConfig.WaitBounds.Max
+				}
+			}
+		}
 	}
 
-	// Setup the Consul config
+	// Set up the Consul config
 	if cc.ConsulConfig != nil {
 		conf.Consul.Address = &cc.ConsulConfig.Addr
 		conf.Consul.Token = &cc.ConsulConfig.Token
@@ -675,6 +731,19 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 				Password: &parts[1],
 			}
 		}
+
+		// Set the user-specified Consul RetryConfig
+		if cc.TemplateConfig.ConsulRetry != nil {
+			var err error
+			err = cc.TemplateConfig.ConsulRetry.Validate()
+			if err != nil {
+				return nil, err
+			}
+			conf.Consul.Retry, err = cc.TemplateConfig.ConsulRetry.ToConsulTemplate()
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	// Get the Consul namespace from job/group config. This is the higher level
@@ -683,7 +752,7 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 		conf.Consul.Namespace = &config.ConsulNamespace
 	}
 
-	// Setup the Vault config
+	// Set up the Vault config
 	// Always set these to ensure nothing is picked up from the environment
 	emptyStr := ""
 	conf.Vault.RenewToken = helper.BoolToPtr(false)
@@ -724,6 +793,18 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 				ServerName: &emptyStr,
 			}
 		}
+
+		// Set the user-specified Vault RetryConfig
+		if cc.TemplateConfig.VaultRetry != nil {
+			var err error
+			if err = cc.TemplateConfig.VaultRetry.Validate(); err != nil {
+				return nil, err
+			}
+			conf.Vault.Retry, err = cc.TemplateConfig.VaultRetry.ToConsulTemplate()
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	conf.Finalize()