From a2868a23f88967a0e6d11c2cdc2b21f341a978e2 Mon Sep 17 00:00:00 2001 From: Derek Strickland <1111455+DerekStrickland@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:08:25 -0400 Subject: [PATCH] consul-template: Add fault tolerant defaults (#13041) consul-template: Add fault tolerant defaults Co-authored-by: Tim Gross --- .changelog/13041.txt | 3 ++ client/config/config.go | 18 +++++-- website/content/docs/configuration/client.mdx | 50 +++++++++++-------- 3 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 .changelog/13041.txt diff --git a/.changelog/13041.txt b/.changelog/13041.txt new file mode 100644 index 000000000000..53f0577f7a1f --- /dev/null +++ b/.changelog/13041.txt @@ -0,0 +1,3 @@ +```release-note:improvement +client: added more fault tolerant defaults for template configuration +``` diff --git a/client/config/config.go b/client/config/config.go index c665604198b5..01d843e029b3 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -64,7 +64,7 @@ var ( "/run/systemd/resolve": "/run/systemd/resolve", } - DefaultTemplateMaxStale = 5 * time.Second + DefaultTemplateMaxStale = 87600 * time.Hour DefaultTemplateFunctionDenylist = []string{"plugin", "writeToFile"} ) @@ -721,8 +721,20 @@ func DefaultConfig() *Config { NoHostUUID: true, DisableRemoteExec: false, TemplateConfig: &ClientTemplateConfig{ - FunctionDenylist: DefaultTemplateFunctionDenylist, - DisableSandbox: false, + FunctionDenylist: DefaultTemplateFunctionDenylist, + DisableSandbox: false, + BlockQueryWaitTime: helper.TimeToPtr(5 * time.Minute), // match Consul default + MaxStale: helper.TimeToPtr(DefaultTemplateMaxStale), // match Consul default + Wait: &WaitConfig{ + Min: helper.TimeToPtr(5 * time.Second), + Max: helper.TimeToPtr(4 * time.Minute), + }, + ConsulRetry: &RetryConfig{ + Attempts: helper.IntToPtr(0), // unlimited + }, + VaultRetry: &RetryConfig{ + Attempts: helper.IntToPtr(0), // unlimited + }, }, RPCHoldTimeout: 5 * time.Second, CNIPath: "/opt/cni/bin", diff --git a/website/content/docs/configuration/client.mdx b/website/content/docs/configuration/client.mdx index 26e4581ebac3..962c27ed548a 100644 --- a/website/content/docs/configuration/client.mdx +++ b/website/content/docs/configuration/client.mdx @@ -246,27 +246,35 @@ chroot as doing so would cause infinite recursion. files on the client host via the `file` function. By default, templates can access files only within the [task working directory]. -- `max_stale` `(string: "")` - # This is the maximum interval to allow "stale" +- `max_stale` `(string: "87600h")` - This is the maximum interval to allow "stale" data. By default, only the Consul leader will respond to queries. Requests to a follower will forward to the leader. In large clusters with many requests, this is not as scalable. This option allows any follower to respond to a query, so long as the last-replicated data is within this bound. Higher values result in less cluster load, but are more likely to have outdated data. -- `wait` `(Code: nil)` - Defines the minimum and maximum amount of time to wait - for the Consul cluster to reach a consistent state before rendering a template. - This is useful to enable in systems where network connectivity to Consul is degraded, - because it will reduce the number of times a template is rendered. This configuration is - also exposed in the _task template stanza_ to allow overrides per task. +- `wait` `(map: { min = "5s" max = "4m" })` - Defines the minimum and maximum amount + of time to wait before attempting to re-render a template. By default, Consul Template + will loop continually and re-render. If this value is set, Nomad will configure Consul + Template with a backoff timer that will tick on an interval equal to the specified `min` + value. Consul Template will always wait at least the as long as the `min` value specified. + If the underlying data has not changed between two tick intervals, Consul Template will + re-render. If the underlying data has changed, Consul Template will delay re-rendering + until the underlying data stabilizes for at least one tick interval, or the configured + `max` duration has elapsed. Once the `max` duration has elapsed, Consul Template will + re-render the template with the data available at the time. This is useful to enable in + systems where Consul is in a degraded state, or the referenced data values are changing + rapidly, because it will reduce the number of times a template is rendered. This + configuration is also exposed in the _task template stanza_ to allow overrides per task. ```hcl wait { min = "5s" - max = "10s" + max = "4m" } ``` -- `wait_bounds` `(Code: nil)` - Defines client level lower and upper bounds for +- `wait_bounds` `(map: nil)` - Defines client level lower and upper bounds for per-template `wait` configuration. If the individual template configuration has a `min` lower than `wait_bounds.min` or a `max` greater than the `wait_bounds.max`, the bounds will be enforced, and the template `wait` will be adjusted before being @@ -279,23 +287,23 @@ chroot as doing so would cause infinite recursion. } ``` -- `block_query_wait` `(string: "60s")` - This is amount of time in seconds to wait +- `block_query_wait` `(string: "5m")` - This is amount of time in seconds to wait for the results of a blocking query. Many endpoints in Consul support a feature known as "blocking queries". A blocking query is used to wait for a potential change using long polling. -- `consul_retry` `(Code: nil)` - This controls the retry behavior when an error is - returned from Consul. Consul Template is highly fault tolerant, meaning it does - not exit in the face of failure. Instead, it uses exponential back-off and retry - functions to wait for the cluster to become available, as is customary in distributed - systems. +- `consul_retry` `(map: { attempts = 0 backoff = "250ms" max_backoff = "1m" })`- + This controls the retry behavior when an error is returned from Consul. The template + runner will not exit in the face of failure. Instead, it uses exponential back-off + and retry functions to wait for the Consul cluster to become available, as is + customary in distributed systems. ```hcl consul_retry { # This specifies the number of attempts to make before giving up. Each # attempt adds the exponential backoff sleep time. Setting this to # zero will implement an unlimited number of retries. - attempts = 12 + attempts = 0 # This is the base amount of time to sleep between retry attempts. Each # retry sleeps for an exponent of 2 longer than this base. For 5 retries, # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s. @@ -309,18 +317,18 @@ chroot as doing so would cause infinite recursion. } ``` -- `vault_retry` `(Code: nil)` - This controls the retry behavior when an error is - returned from Vault. Consul Template is highly fault tolerant, meaning it does - not exit in the face of failure. Instead, it uses exponential back-off and retry - functions to wait for the cluster to become available, as is customary in distributed - systems. +- `vault_retry` `(map: { attempts = 0 backoff = "250ms" max_backoff = "1m" })` - + This controls the retry behavior when an error is returned from Vault. Consul + Template is highly fault tolerant, meaning it does not exit in the face of failure. + Instead, it uses exponential back-off and retry functions to wait for the cluster + to become available, as is customary in distributed systems. ```hcl vault_retry { # This specifies the number of attempts to make before giving up. Each # attempt adds the exponential backoff sleep time. Setting this to # zero will implement an unlimited number of retries. - attempts = 12 + attempts = 0 # This is the base amount of time to sleep between retry attempts. Each # retry sleeps for an exponent of 2 longer than this base. For 5 retries, # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s.