From f768c80905c9aceb022a2a172751836a1ef51fb5 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 11 Jan 2023 09:34:28 -0600 Subject: [PATCH] consul: add client configuration for grpc_ca_file (#15701) * [no ci] first pass at plumbing grpc_ca_file * consul: add support for grpc_ca_file for tls grpc connections in consul 1.14+ This PR adds client config to Nomad for specifying consul.grpc_ca_file These changes combined with https://github.com/hashicorp/consul/pull/15913 should finally enable Nomad users to upgrade to Consul 1.14+ and use tls grpc connections. * consul: add cl entgry for grpc_ca_file * docs: mention grpc_tls changes due to Consul 1.14 --- .changelog/15701.txt | 3 ++ client/allocrunner/consul_grpc_sock_hook.go | 1 - .../taskrunner/envoy_bootstrap_hook.go | 42 +++++++++++-------- .../taskrunner/envoy_bootstrap_hook_test.go | 16 +++---- nomad/structs/config/consul.go | 8 ++++ nomad/structs/config/consul_test.go | 3 ++ website/content/docs/configuration/consul.mdx | 4 ++ .../docs/integrations/consul-connect.mdx | 42 +++++++++++++++++++ 8 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 .changelog/15701.txt diff --git a/.changelog/15701.txt b/.changelog/15701.txt new file mode 100644 index 000000000000..bc358d2b1fec --- /dev/null +++ b/.changelog/15701.txt @@ -0,0 +1,3 @@ +```release-note:improvement +consul: add client configuration for grpc_ca_file +``` diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index 55292340acda..6a7b603bda54 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -195,7 +195,6 @@ func (p *grpcSocketProxy) run(alloc *structs.Allocation) error { return fmt.Errorf("error parsing Consul address %q: %v", p.config.Addr, err) } - destAddr = net.JoinHostPort(host, p.consulGRPCFallbackPort) } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go index 02d2c6edce68..10e60e44f2dc 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go @@ -42,27 +42,29 @@ const ( ) type consulTransportConfig struct { - HTTPAddr string // required - Auth string // optional, env CONSUL_HTTP_AUTH - SSL string // optional, env CONSUL_HTTP_SSL - VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY - CAFile string // optional, arg -ca-file - CertFile string // optional, arg -client-cert - KeyFile string // optional, arg -client-key - Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE + HTTPAddr string // required + Auth string // optional, env CONSUL_HTTP_AUTH + SSL string // optional, env CONSUL_HTTP_SSL + VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY + GRPCCAFile string // optional, arg -grpc-ca-file + CAFile string // optional, arg -ca-file + CertFile string // optional, arg -client-cert + KeyFile string // optional, arg -client-key + Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE // CAPath (dir) not supported by Nomad's config object } -func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig { +func newConsulTransportConfig(cc *config.ConsulConfig) consulTransportConfig { return consulTransportConfig{ - HTTPAddr: consul.Addr, - Auth: consul.Auth, - SSL: decodeTriState(consul.EnableSSL), - VerifySSL: decodeTriState(consul.VerifySSL), - CAFile: consul.CAFile, - CertFile: consul.CertFile, - KeyFile: consul.KeyFile, - Namespace: consul.Namespace, + HTTPAddr: cc.Addr, + Auth: cc.Auth, + SSL: decodeTriState(cc.EnableSSL), + VerifySSL: decodeTriState(cc.VerifySSL), + GRPCCAFile: cc.GRPCCAFile, + CAFile: cc.CAFile, + CertFile: cc.CertFile, + KeyFile: cc.KeyFile, + Namespace: cc.Namespace, } } @@ -123,7 +125,7 @@ type envoyBootstrapHook struct { // envoyBootstrapWaitTime is the total amount of time hook will wait for Consul envoyBootstrapWaitTime time.Duration - // envoyBootstrapInitialGap is the initial wait gap when retyring + // envoyBootstrapInitialGap is the initial wait gap when retrying envoyBoostrapInitialGap time.Duration // envoyBootstrapMaxJitter is the maximum amount of jitter applied to retries @@ -533,6 +535,10 @@ func (e envoyBootstrapArgs) args() []string { arguments = append(arguments, "-token", v) } + if v := e.consulConfig.GRPCCAFile; v != "" { + arguments = append(arguments, "-grpc-ca-file", v) + } + if v := e.consulConfig.CAFile; v != "" { arguments = append(arguments, "-ca-file", v) } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go index 14c1115b06f0..56f0a6f73af5 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go @@ -108,13 +108,14 @@ var ( } consulTLSConfig = consulTransportConfig{ - HTTPAddr: "2.2.2.2", // arg - Auth: "user:password", // env - SSL: "true", // env - VerifySSL: "true", // env - CAFile: "/etc/tls/ca-file", // arg - CertFile: "/etc/tls/cert-file", // arg - KeyFile: "/etc/tls/key-file", // arg + HTTPAddr: "2.2.2.2", // arg + Auth: "user:password", // env + SSL: "true", // env + VerifySSL: "true", // env + GRPCCAFile: "/etc/tls/grpc-ca-file", // arg + CAFile: "/etc/tls/ca-file", // arg + CertFile: "/etc/tls/cert-file", // arg + KeyFile: "/etc/tls/key-file", // arg } ) @@ -178,6 +179,7 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) { "-address", "127.0.0.1:19100", "-bootstrap", "-sidecar-for", "s1", + "-grpc-ca-file", "/etc/tls/grpc-ca-file", "-ca-file", "/etc/tls/ca-file", "-client-cert", "/etc/tls/cert-file", "-client-key", "/etc/tls/key-file", diff --git a/nomad/structs/config/consul.go b/nomad/structs/config/consul.go index e5caf3fdbfce..1cbc81d43c7a 100644 --- a/nomad/structs/config/consul.go +++ b/nomad/structs/config/consul.go @@ -100,6 +100,11 @@ type ConsulConfig struct { // Uses Consul's default and env var. VerifySSL *bool `hcl:"verify_ssl"` + // GRPCCAFile is the path to the ca certificate used for Consul gRPC communication. + // + // Uses Consul's default and env var. + GRPCCAFile string `hcl:"grpc_ca_file"` + // CAFile is the path to the ca certificate used for Consul communication. // // Uses Consul's default and env var. @@ -217,6 +222,9 @@ func (c *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig { if b.ShareSSL != nil { result.ShareSSL = pointer.Of(*b.ShareSSL) } + if b.GRPCCAFile != "" { + result.GRPCCAFile = b.GRPCCAFile + } if b.CAFile != "" { result.CAFile = b.CAFile } diff --git a/nomad/structs/config/consul_test.go b/nomad/structs/config/consul_test.go index 4a8c56015999..bfc287ba3ed6 100644 --- a/nomad/structs/config/consul_test.go +++ b/nomad/structs/config/consul_test.go @@ -53,6 +53,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "1", EnableSSL: &no, VerifySSL: &no, + GRPCCAFile: "1", CAFile: "1", CertFile: "1", KeyFile: "1", @@ -80,6 +81,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", @@ -107,6 +109,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", diff --git a/website/content/docs/configuration/consul.mdx b/website/content/docs/configuration/consul.mdx index 17e92c6ec9bb..1d848d462dc9 100644 --- a/website/content/docs/configuration/consul.mdx +++ b/website/content/docs/configuration/consul.mdx @@ -60,6 +60,10 @@ configuring Nomad to talk to Consul via DNS such as consul.service.consul respective services, each tagged appropriately with either `http` or `rpc` tag. Nomad servers also advertise a `serf` tagged service. +- `grpc_ca_file` `(string: "")` - Specifies an optional path to the GRPC CA + certificate used for communication between Connect sidecar proxies and Consul + agents. Will default to the `CONSUL_GRPC_CACERT` environment variable if set. + - `ca_file` `(string: "")` - Specifies an optional path to the CA certificate used for Consul communication. This defaults to the system bundle if unspecified. Will default to the `CONSUL_CACERT` environment variable if set. diff --git a/website/content/docs/integrations/consul-connect.mdx b/website/content/docs/integrations/consul-connect.mdx index 218b18a04c79..60a2e31ac07d 100644 --- a/website/content/docs/integrations/consul-connect.mdx +++ b/website/content/docs/integrations/consul-connect.mdx @@ -99,6 +99,44 @@ For JSON configurations: } ``` +#### Consul TLS + +~> **Note:** Consul 1.14+ made a [backwards incompatible change][consul_grpc_tls] +in how TLS enabled grpc listeners work. When using Consul 1.14 with TLS enabled users +will need to specify additional Nomad agent configuration to work with Connect. The +`consul.grpc_ca_file` value must now be configured (introduced in Nomad 1.4.4), +and `consul.grpc_address` will most likely need to be set to use the new standard +`grpc_tls` port of `8503`. + +```hcl +consul { + grpc_ca_file = "/etc/tls/consul-agent-ca.pem" + grpc_address = "127.0.0.1:8503" + ca_file = "/etc/tls/consul-agent-ca.pem" + cert_file = "/etc/tls/dc1-client-consul-0.pem" + key_file = "/etc/tls/dc1-client-consul-0-key.pem" + ssl = true + address = "127.0.0.1:8501" +} +``` + +#### Consul ACLs + +~> **Note:** Starting in Nomad v1.3.0, Consul Service Identity ACL tokens automatically +generated by Nomad on behalf of Connect enabled services are now created in [`Local`] +rather than Global scope, and are no longer replicated globally. + +To facilitate cross-Consul datacenter requests of Connect services registered by +Nomad, Consul agents will need to be configured with [default anonymous][anon_token] +ACL tokens with ACL policies of sufficient permissions to read service and node +metadata pertaining to those requests. This mechanism is described in Consul [#7414][consul_acl]. +A typical Consul agent anonymous token may contain an ACL policy such as: + +```hcl +service_prefix "" { policy = "read" } +node_prefix "" { policy = "read" } +``` + ### Nomad Nomad must schedule onto a routable interface in order for the proxies to @@ -340,4 +378,8 @@ dashes (`-`) are converted to underscores (`_`) in environment variables so [`Local`]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#token-attributes [anon_token]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#special-purpose-tokens [consul_ports]: https://developer.hashicorp.com/consul/docs/agent/config/config-files#ports +<<<<<<< HEAD +======= +[consul_grpc_tls]: https://developer.hashicorp.com/consul/docs/upgrading/upgrade-specific#changes-to-grpc-tls-configuration +>>>>>>> c3017da6a (consul: add client configuration for grpc_ca_file (#15701))