consul-template: Add fault tolerant defaults (#13041)

consul-template: Add fault tolerant defaults

Co-authored-by: Tim Gross <tgross@hashicorp.com>

.changelog/13041.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+client: added more fault tolerant defaults for template configuration
+```

client/config/config.go

@@ -64,7 +64,7 @@ var (
 		"/run/systemd/resolve": "/run/systemd/resolve",
 	}
 
-	DefaultTemplateMaxStale = 5 * time.Second
+	DefaultTemplateMaxStale = 87600 * time.Hour
 
 	DefaultTemplateFunctionDenylist = []string{"plugin", "writeToFile"}
 )
@@ -721,8 +721,20 @@ func DefaultConfig() *Config {
 		NoHostUUID:              true,
 		DisableRemoteExec:       false,
 		TemplateConfig: &ClientTemplateConfig{
-			FunctionDenylist: DefaultTemplateFunctionDenylist,
-			DisableSandbox:   false,
+			FunctionDenylist:   DefaultTemplateFunctionDenylist,
+			DisableSandbox:     false,
+			BlockQueryWaitTime: helper.TimeToPtr(5 * time.Minute),         // match Consul default
+			MaxStale:           helper.TimeToPtr(DefaultTemplateMaxStale), // match Consul default
+			Wait: &WaitConfig{
+				Min: helper.TimeToPtr(5 * time.Second),
+				Max: helper.TimeToPtr(4 * time.Minute),
+			},
+			ConsulRetry: &RetryConfig{
+				Attempts: helper.IntToPtr(0), // unlimited
+			},
+			VaultRetry: &RetryConfig{
+				Attempts: helper.IntToPtr(0), // unlimited
+			},
 		},
 		RPCHoldTimeout:     5 * time.Second,
 		CNIPath:            "/opt/cni/bin",

website/content/docs/configuration/client.mdx

@@ -246,27 +246,35 @@ chroot as doing so would cause infinite recursion.
   files on the client host via the `file` function. By default, templates can
   access files only within the [task working directory].
 
-- `max_stale` `(string: "")` - # This is the maximum interval to allow "stale"
+- `max_stale` `(string: "87600h")` - This is the maximum interval to allow "stale"
   data. By default, only the Consul leader will respond to queries. Requests to
   a follower will forward to the leader. In large clusters with many requests,
   this is not as scalable. This option allows any follower to respond to a query,
   so long as the last-replicated data is within this bound. Higher values result
   in less cluster load, but are more likely to have outdated data.
 
-- `wait` `(Code: nil)` - Defines the minimum and maximum amount of time to wait
-  for the Consul cluster to reach a consistent state before rendering a template.
-  This is useful to enable in systems where network connectivity to Consul is degraded,
-  because it will reduce the number of times a template is rendered. This configuration is
-  also exposed in the _task template stanza_ to allow overrides per task.
+- `wait` `(map: { min = "5s" max = "4m" })` - Defines the minimum and maximum amount
+  of time to wait before attempting to re-render a template. By default, Consul Template
+  will loop continually and re-render. If this value is set, Nomad will configure Consul
+  Template with a backoff timer that will tick on an interval equal to the specified `min`
+  value. Consul Template will always wait at least the as long as the `min` value specified.
+  If the underlying data has not changed between two tick intervals, Consul Template will
+  re-render. If the underlying data has changed, Consul Template will delay re-rendering
+  until the underlying data stabilizes for at least one tick interval, or the configured
+  `max` duration has elapsed. Once the `max` duration has elapsed, Consul Template will
+  re-render the template with the data available at the time. This is useful to enable in
+  systems where Consul is in a degraded state, or the referenced data values are changing
+  rapidly, because it will reduce the number of times a template is rendered. This
+  configuration is also exposed in the _task template stanza_ to allow overrides per task.
 
   ```hcl
   wait {
     min     = "5s"
-    max     = "10s"
+    max     = "4m"
   }
   ```
 
-- `wait_bounds` `(Code: nil)` - Defines client level lower and upper bounds for
+- `wait_bounds` `(map: nil)` - Defines client level lower and upper bounds for
   per-template `wait` configuration. If the individual template configuration has
   a `min` lower than `wait_bounds.min` or a `max` greater than the `wait_bounds.max`,
   the bounds will be enforced, and the template `wait` will be adjusted before being
@@ -279,23 +287,23 @@ chroot as doing so would cause infinite recursion.
   }
     ```
 
-- `block_query_wait` `(string: "60s")` - This is amount of time in seconds to wait
+- `block_query_wait` `(string: "5m")` - This is amount of time in seconds to wait
   for the results of a blocking query. Many endpoints in Consul support a feature known as
   "blocking queries". A blocking query is used to wait for a potential change
   using long polling.
 
-- `consul_retry` `(Code: nil)` - This controls the retry behavior when an error is
-  returned from Consul. Consul Template is highly fault tolerant, meaning it does
-  not exit in the face of failure. Instead, it uses exponential back-off and retry
-  functions to wait for the cluster to become available, as is customary in distributed
-  systems.
+- `consul_retry` `(map: { attempts = 0 backoff = "250ms" max_backoff = "1m" })`-
+  This controls the retry behavior when an error is returned from Consul. The template
+  runner will not exit in the face of failure. Instead, it uses exponential back-off
+  and retry functions to wait for the Consul cluster to become available, as is
+  customary in distributed systems.
 
   ```hcl
   consul_retry {
     # This specifies the number of attempts to make before giving up. Each
     # attempt adds the exponential backoff sleep time. Setting this to
     # zero will implement an unlimited number of retries.
-    attempts = 12
+    attempts = 0
     # This is the base amount of time to sleep between retry attempts. Each
     # retry sleeps for an exponent of 2 longer than this base. For 5 retries,
     # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s.
@@ -309,18 +317,18 @@ chroot as doing so would cause infinite recursion.
   }
   ```
 
-- `vault_retry` `(Code: nil)` - This controls the retry behavior when an error is
-  returned from Vault. Consul Template is highly fault tolerant, meaning it does
-  not exit in the face of failure. Instead, it uses exponential back-off and retry
-  functions to wait for the cluster to become available, as is customary in distributed
-  systems.
+- `vault_retry` `(map: { attempts = 0 backoff = "250ms" max_backoff = "1m" })` -
+  This controls the retry behavior when an error is returned from Vault. Consul
+  Template is highly fault tolerant, meaning it does not exit in the face of failure.
+  Instead, it uses exponential back-off and retry functions to wait for the cluster
+  to become available, as is customary in distributed systems.
 
   ```hcl
   vault_retry {
     # This specifies the number of attempts to make before giving up. Each
     # attempt adds the exponential backoff sleep time. Setting this to
     # zero will implement an unlimited number of retries.
-    attempts = 12
+    attempts = 0
     # This is the base amount of time to sleep between retry attempts. Each
     # retry sleeps for an exponent of 2 longer than this base. For 5 retries,
     # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s.