From b527eb2b20447b7c627c74d4d781d72dbbe74f3b Mon Sep 17 00:00:00 2001 From: Derek Strickland Date: Sat, 23 Jul 2022 07:20:01 -0400 Subject: [PATCH 1/4] add Nomad RetryConfig to agent template config --- .../taskrunner/template/template.go | 11 ++++++++++ .../taskrunner/template/template_test.go | 13 +++++++++++ client/config/config.go | 22 ++++++++++++++++--- command/agent/config_parse.go | 15 +++++++++++++ command/agent/config_test.go | 6 +++++ .../test-resources/client_with_template.hcl | 6 +++++ 6 files changed, 70 insertions(+), 3 deletions(-) diff --git a/client/allocrunner/taskrunner/template/template.go b/client/allocrunner/taskrunner/template/template.go index 3d4308c707d9..3eed8034e818 100644 --- a/client/allocrunner/taskrunner/template/template.go +++ b/client/allocrunner/taskrunner/template/template.go @@ -823,6 +823,17 @@ func newRunnerConfig(config *TaskTemplateManagerConfig, conf.Nomad.Namespace = &config.NomadNamespace conf.Nomad.Transport.CustomDialer = cc.TemplateDialer conf.Nomad.Token = &config.NomadToken + if cc.TemplateConfig != nil && cc.TemplateConfig.NomadRetry != nil { + // Set the user-specified Nomad RetryConfig + var err error + if err = cc.TemplateConfig.NomadRetry.Validate(); err != nil { + return nil, err + } + conf.Nomad.Retry, err = cc.TemplateConfig.NomadRetry.ToConsulTemplate() + if err != nil { + return nil, err + } + } conf.Finalize() return conf, nil diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index 2c679ba1fae0..b22fe7e1881c 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1959,6 +1959,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { clientConfig.TemplateConfig.Wait = waitConfig.Copy() clientConfig.TemplateConfig.ConsulRetry = retryConfig.Copy() clientConfig.TemplateConfig.VaultRetry = retryConfig.Copy() + clientConfig.TemplateConfig.NomadRetry = retryConfig.Copy() alloc := mock.Alloc() allocWithOverride := mock.Alloc() @@ -1986,6 +1987,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { Wait: waitConfig.Copy(), ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, &TaskTemplateManagerConfig{ ClientConfig: clientConfig, @@ -1999,6 +2001,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { Wait: waitConfig.Copy(), ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, }, &templateconfig.TemplateConfig{ @@ -2017,6 +2020,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { Wait: waitConfig.Copy(), ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, &TaskTemplateManagerConfig{ ClientConfig: clientConfig, @@ -2030,6 +2034,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { Wait: waitConfig.Copy(), ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, }, &templateconfig.TemplateConfig{ @@ -2052,6 +2057,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { }, ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, &TaskTemplateManagerConfig{ ClientConfig: clientConfig, @@ -2077,6 +2083,7 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { }, ConsulRetry: retryConfig.Copy(), VaultRetry: retryConfig.Copy(), + NomadRetry: retryConfig.Copy(), }, }, &templateconfig.TemplateConfig{ @@ -2117,6 +2124,12 @@ func TestTaskTemplateManager_ClientTemplateConfig_Set(t *testing.T) { require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.VaultRetry.Attempts, *runnerConfig.Vault.Retry.Attempts) require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.VaultRetry.Backoff, *runnerConfig.Vault.Retry.Backoff) require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.VaultRetry.MaxBackoff, *runnerConfig.Vault.Retry.MaxBackoff) + // Nomad Retry + require.NotNil(t, runnerConfig.Nomad) + require.NotNil(t, runnerConfig.Nomad.Retry) + require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.NomadRetry.Attempts, *runnerConfig.Nomad.Retry.Attempts) + require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.NomadRetry.Backoff, *runnerConfig.Nomad.Retry.Backoff) + require.Equal(t, *_case.ExpectedRunnerConfig.TemplateConfig.NomadRetry.MaxBackoff, *runnerConfig.Nomad.Retry.MaxBackoff) // Test that wait_bounds are enforced for _, tmpl := range *runnerConfig.Templates { diff --git a/client/config/config.go b/client/config/config.go index 01d843e029b3..625b297bf54f 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "github.com/hashicorp/nomad/helper/pointer" "io" "os" "reflect" @@ -358,6 +359,13 @@ type ClientTemplateConfig struct { // to wait for the cluster to become available, as is customary in distributed // systems. VaultRetry *RetryConfig `hcl:"vault_retry,optional"` + + // This controls the retry behavior when an error is returned from Nomad. + // Consul Template is highly fault tolerant, meaning it does not exit in the + // face of failure. Instead, it uses exponential back-off and retry functions + // to wait for the cluster to become available, as is customary in distributed + // systems. + NomadRetry *RetryConfig `hcl:"nomad_retry,optional"` } // Copy returns a deep copy of a ClientTemplateConfig @@ -396,6 +404,10 @@ func (c *ClientTemplateConfig) Copy() *ClientTemplateConfig { nc.VaultRetry = c.VaultRetry.Copy() } + if c.NomadRetry != nil { + nc.NomadRetry = c.NomadRetry.Copy() + } + return nc } @@ -413,7 +425,8 @@ func (c *ClientTemplateConfig) IsEmpty() bool { c.MaxStaleHCL == "" && c.Wait.IsEmpty() && c.ConsulRetry.IsEmpty() && - c.VaultRetry.IsEmpty() + c.VaultRetry.IsEmpty() && + c.NomadRetry.IsEmpty() } // WaitConfig is mirrored from templateconfig.WaitConfig because we need to handle @@ -730,10 +743,13 @@ func DefaultConfig() *Config { Max: helper.TimeToPtr(4 * time.Minute), }, ConsulRetry: &RetryConfig{ - Attempts: helper.IntToPtr(0), // unlimited + Attempts: pointer.Of[int](0), // unlimited }, VaultRetry: &RetryConfig{ - Attempts: helper.IntToPtr(0), // unlimited + Attempts: pointer.Of[int](0), // unlimited + }, + NomadRetry: &RetryConfig{ + Attempts: pointer.Of[int](0), // unlimited }, }, RPCHoldTimeout: 5 * time.Second, diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 04135f78324d..4f7243d33f33 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -41,6 +41,7 @@ func ParseConfigFile(path string) (*Config, error) { WaitBounds: &client.WaitConfig{}, ConsulRetry: &client.RetryConfig{}, VaultRetry: &client.RetryConfig{}, + NomadRetry: &client.RetryConfig{}, }, }, Server: &ServerConfig{ @@ -125,6 +126,16 @@ func ParseConfigFile(path string) (*Config, error) { c.Client.TemplateConfig.VaultRetry.MaxBackoff = d }, }, + {"client.template.nomad_retry.backoff", nil, &c.Client.TemplateConfig.NomadRetry.BackoffHCL, + func(d *time.Duration) { + c.Client.TemplateConfig.NomadRetry.Backoff = d + }, + }, + {"client.template.nomad_retry.max_backoff", nil, &c.Client.TemplateConfig.NomadRetry.MaxBackoffHCL, + func(d *time.Duration) { + c.Client.TemplateConfig.NomadRetry.MaxBackoff = d + }, + }, } // Add enterprise audit sinks for time.Duration parsing @@ -264,6 +275,10 @@ func finalizeClientTemplateConfig(config *Config) { config.Client.TemplateConfig.VaultRetry = nil } + if config.Client.TemplateConfig.NomadRetry.IsEmpty() { + config.Client.TemplateConfig.NomadRetry = nil + } + if config.Client.TemplateConfig.IsEmpty() { config.Client.TemplateConfig = nil } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 2e98438f382a..f88c775ed725 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -1432,6 +1432,7 @@ func TestConfig_LoadConsulTemplateConfig(t *testing.T) { require.NotNil(t, templateConfig.WaitBounds) require.NotNil(t, templateConfig.ConsulRetry) require.NotNil(t, templateConfig.VaultRetry) + require.NotNil(t, templateConfig.NomadRetry) // Direct properties require.Equal(t, 300*time.Second, *templateConfig.MaxStale) @@ -1452,6 +1453,11 @@ func TestConfig_LoadConsulTemplateConfig(t *testing.T) { require.Equal(t, 10, *templateConfig.VaultRetry.Attempts) require.Equal(t, 15*time.Second, *templateConfig.VaultRetry.Backoff) require.Equal(t, 20*time.Second, *templateConfig.VaultRetry.MaxBackoff) + // Nomad Retry + require.NotNil(t, templateConfig.NomadRetry) + require.Equal(t, 15, *templateConfig.NomadRetry.Attempts) + require.Equal(t, 20*time.Second, *templateConfig.NomadRetry.Backoff) + require.Equal(t, 25*time.Second, *templateConfig.NomadRetry.MaxBackoff) } func TestConfig_LoadConsulTemplate_FunctionDenylist(t *testing.T) { diff --git a/command/agent/test-resources/client_with_template.hcl b/command/agent/test-resources/client_with_template.hcl index f094cda437e7..ff310f22f8b4 100644 --- a/command/agent/test-resources/client_with_template.hcl +++ b/command/agent/test-resources/client_with_template.hcl @@ -26,6 +26,12 @@ client { backoff = "15s" max_backoff = "20s" } + + nomad_retry { + attempts = 15 + backoff = "20s" + max_backoff = "25s" + } } } From 198a25c91df7270f320e67f8733dda0cee08b91b Mon Sep 17 00:00:00 2001 From: Derek Strickland Date: Mon, 1 Aug 2022 14:07:02 -0400 Subject: [PATCH 2/4] Add docs and changelog entry --- .changelog/13907.txt | 3 +++ website/content/docs/configuration/client.mdx | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .changelog/13907.txt diff --git a/.changelog/13907.txt b/.changelog/13907.txt new file mode 100644 index 000000000000..50dc4a0ce192 --- /dev/null +++ b/.changelog/13907.txt @@ -0,0 +1,3 @@ +```release-note:improvement +template: Expose consul-template configuration options at the client level for `nomad_retry`. +``` diff --git a/website/content/docs/configuration/client.mdx b/website/content/docs/configuration/client.mdx index cfec4d515e26..6db9ded601a7 100644 --- a/website/content/docs/configuration/client.mdx +++ b/website/content/docs/configuration/client.mdx @@ -347,6 +347,31 @@ chroot as doing so would cause infinite recursion. } ``` +- `nomad_retry` `(map: { attempts = 0 backoff = "250ms" max_backoff = "1m" })` - + This controls the retry behavior when an error is returned from Nomad. Consul + Template is highly fault tolerant, meaning it does not exit in the face of failure. + Instead, it uses exponential back-off and retry functions to wait for the cluster + to become available, as is customary in distributed systems. + + ```hcl + nomad_retry { + # This specifies the number of attempts to make before giving up. Each + # attempt adds the exponential backoff sleep time. Setting this to + # zero will implement an unlimited number of retries. + attempts = 0 + # This is the base amount of time to sleep between retry attempts. Each + # retry sleeps for an exponent of 2 longer than this base. For 5 retries, + # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s. + backoff = "250ms" + # This is the maximum amount of time to sleep between retry attempts. + # When max_backoff is set to zero, there is no upper limit to the + # exponential sleep between retry attempts. + # If max_backoff is set to 10s and backoff is set to 1s, sleep times + # would be: 1s, 2s, 4s, 8s, 10s, 10s, ... + max_backoff = "1m" + } + ``` + ### `host_volume` Stanza The `host_volume` stanza is used to make volumes available to jobs. From 9a0f463f060852dab3d839705dcc9123b9c67d81 Mon Sep 17 00:00:00 2001 From: Derek Strickland Date: Wed, 3 Aug 2022 14:13:31 -0400 Subject: [PATCH 3/4] update consul template dependency --- go.mod | 6 +++--- go.sum | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c1ddc2fd8cf5..5a777193a3cf 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/gosuri/uilive v0.0.4 github.com/grpc-ecosystem/go-grpc-middleware v1.2.1-0.20200228141219-3ce3d519df39 github.com/hashicorp/consul v1.7.8 - github.com/hashicorp/consul-template v0.29.1 + github.com/hashicorp/consul-template v0.29.2-0.20220803104536-583050a85eea github.com/hashicorp/consul/api v1.13.0 github.com/hashicorp/consul/sdk v0.9.0 github.com/hashicorp/cronexpr v1.1.1 @@ -76,7 +76,7 @@ require ( github.com/hashicorp/logutils v1.0.0 github.com/hashicorp/memberlist v0.3.1 github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 - github.com/hashicorp/nomad/api v0.0.0-20220407202126-2eba643965c4 + github.com/hashicorp/nomad/api v0.0.0-20220707195938-75f4c2237b28 github.com/hashicorp/raft v1.3.9 github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.7 @@ -214,7 +214,7 @@ require ( github.com/hashicorp/vault/api/auth/kubernetes v0.1.0 // indirect github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 // indirect github.com/huandu/xstrings v1.3.2 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 // indirect github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index 0dfc011d5b08..f739d8ca5d49 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul v1.7.8 h1:hp308KxAf3zWoGuwp2e+0UUhrm6qHjeBQk3jCZ+bjcY= github.com/hashicorp/consul v1.7.8/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDHpz+Q0nLw= -github.com/hashicorp/consul-template v0.29.1 h1:icm/H7klHYlxpUoWqSmTIWaSLEfGqUJJBsZA/2JhTLU= -github.com/hashicorp/consul-template v0.29.1/go.mod h1:QIohwBuXlKXtsmGGQdWrISlUy4E6LFg5tLZyrw4MyoU= +github.com/hashicorp/consul-template v0.29.2-0.20220803104536-583050a85eea h1:d9frD3+sqQOG/4hOXLEfcXnNz+au0owaRUmM2WuzCBk= +github.com/hashicorp/consul-template v0.29.2-0.20220803104536-583050a85eea/go.mod h1:i2oqMe0jIyHAKuimz7Q3sJU3vnwVx3QzDdDmrRrz5RI= github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= github.com/hashicorp/consul/api v1.13.0 h1:2hnLQ0GjQvw7f3O61jMO8gbasZviZTrt9R8WzgiirHc= github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= @@ -838,8 +838,9 @@ github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 h1:rw3IAne6CDuVFlZbPOkA7bhxlqawFh7RJJ+CejfMaxE= github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= @@ -1879,6 +1880,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= From 0808f6c3b482e2027cea60dc63f2ffd1df2b4f5c Mon Sep 17 00:00:00 2001 From: Derek Strickland Date: Wed, 3 Aug 2022 14:51:36 -0400 Subject: [PATCH 4/4] fix goimports lint check --- client/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/config/config.go b/client/config/config.go index 625b297bf54f..4f840064dda4 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -3,7 +3,6 @@ package config import ( "errors" "fmt" - "github.com/hashicorp/nomad/helper/pointer" "io" "os" "reflect" @@ -20,6 +19,7 @@ import ( "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/bufconndialer" "github.com/hashicorp/nomad/helper/pluginutils/loader" + "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/nomad/structs" structsc "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/plugins/base"