From 3dd061bb77fcff14ac5152f431fb8da0da436f9d Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 5 Jan 2023 15:06:03 +0000 Subject: [PATCH 1/4] backport of commit d47218459f8d82d6b1570bce7670d87d6402a251 --- client/allocrunner/consul_grpc_sock_hook.go | 4 +- .../taskrunner/envoy_bootstrap_hook.go | 72 +++++++++---------- .../taskrunner/envoy_bootstrap_hook_test.go | 16 +++-- nomad/structs/config/consul.go | 8 +++ nomad/structs/config/consul_test.go | 3 + website/content/docs/configuration/consul.mdx | 4 ++ 6 files changed, 60 insertions(+), 47 deletions(-) diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index a9e7c2ff1b4a..1e683cb107b8 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -60,6 +60,7 @@ func newConsulGRPCSocketHook( if !ok { consulGRPCPort = consulGRPCFallbackPort } + logger.Info("CONSUL GRPC DEBUG", "consulGRPCPort", consulGRPCPort) return &consulGRPCSocketHook{ alloc: alloc, @@ -195,7 +196,6 @@ func (p *grpcSocketProxy) run(alloc *structs.Allocation) error { return fmt.Errorf("error parsing Consul address %q: %v", p.config.Addr, err) } - destAddr = net.JoinHostPort(host, p.consulGRPCFallbackPort) } @@ -302,6 +302,8 @@ func proxyConn(ctx context.Context, logger hclog.Logger, destAddr string, conn n // Close the connection when we're done with it. defer conn.Close() + logger.Info("CONSUL GRPC DEBUG", "destAddr", destAddr) + ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go index 0445d3a2e240..8848aadda6e0 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go @@ -44,27 +44,29 @@ const ( ) type consulTransportConfig struct { - HTTPAddr string // required - Auth string // optional, env CONSUL_HTTP_AUTH - SSL string // optional, env CONSUL_HTTP_SSL - VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY - CAFile string // optional, arg -ca-file - CertFile string // optional, arg -client-cert - KeyFile string // optional, arg -client-key - Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE + HTTPAddr string // required + Auth string // optional, env CONSUL_HTTP_AUTH + SSL string // optional, env CONSUL_HTTP_SSL + VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY + GRPCCAFile string // optional, arg -grpc-ca-file + CAFile string // optional, arg -ca-file + CertFile string // optional, arg -client-cert + KeyFile string // optional, arg -client-key + Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE // CAPath (dir) not supported by Nomad's config object } -func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig { +func newConsulTransportConfig(cc *config.ConsulConfig) consulTransportConfig { return consulTransportConfig{ - HTTPAddr: consul.Addr, - Auth: consul.Auth, - SSL: decodeTriState(consul.EnableSSL), - VerifySSL: decodeTriState(consul.VerifySSL), - CAFile: consul.CAFile, - CertFile: consul.CertFile, - KeyFile: consul.KeyFile, - Namespace: consul.Namespace, + HTTPAddr: cc.Addr, + Auth: cc.Auth, + SSL: decodeTriState(cc.EnableSSL), + VerifySSL: decodeTriState(cc.VerifySSL), + GRPCCAFile: cc.GRPCCAFile, + CAFile: cc.CAFile, + CertFile: cc.CertFile, + KeyFile: cc.KeyFile, + Namespace: cc.Namespace, } } @@ -125,7 +127,7 @@ type envoyBootstrapHook struct { // envoyBootstrapWaitTime is the total amount of time hook will wait for Consul envoyBootstrapWaitTime time.Duration - // envoyBootstrapInitialGap is the initial wait gap when retyring + // envoyBootstrapInitialGap is the initial wait gap when retrying envoyBoostrapInitialGap time.Duration // envoyBootstrapMaxJitter is the maximum amount of jitter applied to retries @@ -280,6 +282,8 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart // Create command line arguments bootstrapArgs := bootstrap.args() + h.logger.Info("CONSUL GRPC DEBUG", "bootstrap args", bootstrapArgs) + // Write args to file for debugging argsFile, err := os.Create(bootstrapCmdPath) if err != nil { @@ -542,29 +546,19 @@ func (e envoyBootstrapArgs) args() []string { "-bootstrap", } - if v := e.gateway; v != "" { - arguments = append(arguments, "-gateway", v) - } - - if v := e.siToken; v != "" { - arguments = append(arguments, "-token", v) - } - - if v := e.consulConfig.CAFile; v != "" { - arguments = append(arguments, "-ca-file", v) - } - - if v := e.consulConfig.CertFile; v != "" { - arguments = append(arguments, "-client-cert", v) - } - - if v := e.consulConfig.KeyFile; v != "" { - arguments = append(arguments, "-client-key", v) + appendIfSet := func(param, value string) { + if value != "" { + arguments = append(arguments, param, value) + } } - if v := e.namespace; v != "" { - arguments = append(arguments, "-namespace", v) - } + appendIfSet("-gateway", e.gateway) + appendIfSet("-token", e.siToken) + appendIfSet("-grpc-ca-file", e.consulConfig.GRPCCAFile) + appendIfSet("-ca-file", e.consulConfig.CAFile) + appendIfSet("-client-cert", e.consulConfig.CertFile) + appendIfSet("-client-key", e.consulConfig.KeyFile) + appendIfSet("-namespace", e.namespace) return arguments } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go index 70ebe47b59b3..4fcfb623bb68 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go @@ -105,13 +105,14 @@ var ( } consulTLSConfig = consulTransportConfig{ - HTTPAddr: "2.2.2.2", // arg - Auth: "user:password", // env - SSL: "true", // env - VerifySSL: "true", // env - CAFile: "/etc/tls/ca-file", // arg - CertFile: "/etc/tls/cert-file", // arg - KeyFile: "/etc/tls/key-file", // arg + HTTPAddr: "2.2.2.2", // arg + Auth: "user:password", // env + SSL: "true", // env + VerifySSL: "true", // env + GRPCCAFile: "/etc/tls/grpc-ca-file", // arg + CAFile: "/etc/tls/ca-file", // arg + CertFile: "/etc/tls/cert-file", // arg + KeyFile: "/etc/tls/key-file", // arg } ) @@ -175,6 +176,7 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) { "-address", "127.0.0.1:19100", "-proxy-id", "s1-sidecar-proxy", "-bootstrap", + "-grpc-ca-file", "/etc/tls/grpc-ca-file", "-ca-file", "/etc/tls/ca-file", "-client-cert", "/etc/tls/cert-file", "-client-key", "/etc/tls/key-file", diff --git a/nomad/structs/config/consul.go b/nomad/structs/config/consul.go index bddd7947fed0..cddbdb54ae87 100644 --- a/nomad/structs/config/consul.go +++ b/nomad/structs/config/consul.go @@ -102,6 +102,11 @@ type ConsulConfig struct { // Uses Consul's default and env var. VerifySSL *bool `hcl:"verify_ssl"` + // GRPCCAFile is the path to the ca certificate used for Consul gRPC communication. + // + // Uses Consul's default and env var. + GRPCCAFile string `hcl:"grpc_ca_file"` + // CAFile is the path to the ca certificate used for Consul communication. // // Uses Consul's default and env var. @@ -219,6 +224,9 @@ func (c *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig { if b.ShareSSL != nil { result.ShareSSL = pointer.Of(*b.ShareSSL) } + if b.GRPCCAFile != "" { + result.GRPCCAFile = b.GRPCCAFile + } if b.CAFile != "" { result.CAFile = b.CAFile } diff --git a/nomad/structs/config/consul_test.go b/nomad/structs/config/consul_test.go index 62f2fcd862da..806b999d56a5 100644 --- a/nomad/structs/config/consul_test.go +++ b/nomad/structs/config/consul_test.go @@ -54,6 +54,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "1", EnableSSL: &no, VerifySSL: &no, + GRPCCAFile: "1", CAFile: "1", CertFile: "1", KeyFile: "1", @@ -81,6 +82,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", @@ -108,6 +110,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", diff --git a/website/content/docs/configuration/consul.mdx b/website/content/docs/configuration/consul.mdx index 3c3fc3d5c362..9e9e3a867635 100644 --- a/website/content/docs/configuration/consul.mdx +++ b/website/content/docs/configuration/consul.mdx @@ -61,6 +61,10 @@ configuring Nomad to talk to Consul via DNS such as consul.service.consul respective services, each tagged appropriately with either `http` or `rpc` tag. Nomad servers also advertise a `serf` tagged service. +- `grpc_ca_file` `(string: "")` - Specifies an optional path to the GRPC CA + certificate used for communication between Connect sidecar proxies and Consul + agents. Will default to the `CONSUL_GRPC_CACERT` environment variable if set. + - `ca_file` `(string: "")` - Specifies an optional path to the CA certificate used for Consul communication. This defaults to the system bundle if unspecified. Will default to the `CONSUL_CACERT` environment variable if set. From 05b2ad4cd25cfcd160421670b87c78a71e5c9703 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Fri, 6 Jan 2023 15:55:27 +0000 Subject: [PATCH 2/4] backport of commit e5f7f28e7a6ab214e519996984b0bb562a3d2a6f --- client/allocrunner/consul_grpc_sock_hook.go | 3 --- client/allocrunner/taskrunner/envoy_bootstrap_hook.go | 2 -- 2 files changed, 5 deletions(-) diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index 1e683cb107b8..ba68c2ad13d5 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -60,7 +60,6 @@ func newConsulGRPCSocketHook( if !ok { consulGRPCPort = consulGRPCFallbackPort } - logger.Info("CONSUL GRPC DEBUG", "consulGRPCPort", consulGRPCPort) return &consulGRPCSocketHook{ alloc: alloc, @@ -302,8 +301,6 @@ func proxyConn(ctx context.Context, logger hclog.Logger, destAddr string, conn n // Close the connection when we're done with it. defer conn.Close() - logger.Info("CONSUL GRPC DEBUG", "destAddr", destAddr) - ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go index 8848aadda6e0..0da4c3d0d5e3 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go @@ -282,8 +282,6 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart // Create command line arguments bootstrapArgs := bootstrap.args() - h.logger.Info("CONSUL GRPC DEBUG", "bootstrap args", bootstrapArgs) - // Write args to file for debugging argsFile, err := os.Create(bootstrapCmdPath) if err != nil { From da29afc5a1ffe5ca032ef21fef9e9e9d4e32bc42 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 11 Jan 2023 15:10:29 +0000 Subject: [PATCH 3/4] backport of commit 640b9d93db01d1ea23c49181b8d0765b0a23a26d --- .changelog/15701.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/15701.txt diff --git a/.changelog/15701.txt b/.changelog/15701.txt new file mode 100644 index 000000000000..bc358d2b1fec --- /dev/null +++ b/.changelog/15701.txt @@ -0,0 +1,3 @@ +```release-note:improvement +consul: add client configuration for grpc_ca_file +``` From adbb9b59ea4e37d0c9a7507e3d709403631fabdb Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 11 Jan 2023 15:30:19 +0000 Subject: [PATCH 4/4] backport of commit 6a31f39a154c551b9f7736a7702fe624c8bef862 --- .../docs/integrations/consul-connect.mdx | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/website/content/docs/integrations/consul-connect.mdx b/website/content/docs/integrations/consul-connect.mdx index ea8490d4a84a..a970868f09ce 100644 --- a/website/content/docs/integrations/consul-connect.mdx +++ b/website/content/docs/integrations/consul-connect.mdx @@ -96,6 +96,27 @@ For JSON configurations: } ``` +#### Consul TLS + +~> **Note:** Consul 1.14+ made a [backwards incompatible change][consul_grpc_tls] +in how TLS enabled grpc listeners work. When using Consul 1.14 with TLS enabled users +will need to specify additional Nomad agent configuration to work with Connect. The +`consul.grpc_ca_file` value must now be configured (introduced in Nomad 1.4.4), +and `consul.grpc_address` will most likely need to be set to use the new standard +`grpc_tls` port of `8503`. + +```hcl +consul { + grpc_ca_file = "/etc/tls/consul-agent-ca.pem" + grpc_address = "127.0.0.1:8503" + ca_file = "/etc/tls/consul-agent-ca.pem" + cert_file = "/etc/tls/dc1-client-consul-0.pem" + key_file = "/etc/tls/dc1-client-consul-0-key.pem" + ssl = true + address = "127.0.0.1:8501" +} +``` + #### Consul ACLs ~> **Note:** Starting in Nomad v1.3.0, Consul Service Identity ACL tokens automatically @@ -356,3 +377,4 @@ dashes (`-`) are converted to underscores (`_`) in environment variables so [`Local`]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#token-attributes [anon_token]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#special-purpose-tokens [consul_ports]: https://developer.hashicorp.com/consul/docs/agent/config/config-files#ports +[consul_grpc_tls]: https://developer.hashicorp.com/consul/docs/upgrading/upgrade-specific#changes-to-grpc-tls-configuration \ No newline at end of file