consul: add client configuration for grpc_ca_file (#15701)

* [no ci] first pass at plumbing grpc_ca_file

* consul: add support for grpc_ca_file for tls grpc connections in consul 1.14+

This PR adds client config to Nomad for specifying consul.grpc_ca_file

These changes combined with hashicorp/consul#15913 should
finally enable Nomad users to upgrade to Consul 1.14+ and use tls grpc connections.

* consul: add cl entgry for grpc_ca_file

* docs: mention grpc_tls changes due to Consul 1.14

.changelog/15701.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul: add client configuration for grpc_ca_file
+```

client/allocrunner/consul_grpc_sock_hook.go

@@ -195,7 +195,6 @@ func (p *grpcSocketProxy) run(alloc *structs.Allocation) error {
 			return fmt.Errorf("error parsing Consul address %q: %v",
 				p.config.Addr, err)
 		}
-
 		destAddr = net.JoinHostPort(host, p.consulGRPCFallbackPort)
 	}
 

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -44,27 +44,29 @@ const (
 )
 
 type consulTransportConfig struct {
-	HTTPAddr  string // required
-	Auth      string // optional, env CONSUL_HTTP_AUTH
-	SSL       string // optional, env CONSUL_HTTP_SSL
-	VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY
-	CAFile    string // optional, arg -ca-file
-	CertFile  string // optional, arg -client-cert
-	KeyFile   string // optional, arg -client-key
-	Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE
+	HTTPAddr   string // required
+	Auth       string // optional, env CONSUL_HTTP_AUTH
+	SSL        string // optional, env CONSUL_HTTP_SSL
+	VerifySSL  string // optional, env CONSUL_HTTP_SSL_VERIFY
+	GRPCCAFile string // optional, arg -grpc-ca-file
+	CAFile     string // optional, arg -ca-file
+	CertFile   string // optional, arg -client-cert
+	KeyFile    string // optional, arg -client-key
+	Namespace  string // optional, only consul Enterprise, env CONSUL_NAMESPACE
 	// CAPath (dir) not supported by Nomad's config object
 }
 
-func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig {
+func newConsulTransportConfig(cc *config.ConsulConfig) consulTransportConfig {
 	return consulTransportConfig{
-		HTTPAddr:  consul.Addr,
-		Auth:      consul.Auth,
-		SSL:       decodeTriState(consul.EnableSSL),
-		VerifySSL: decodeTriState(consul.VerifySSL),
-		CAFile:    consul.CAFile,
-		CertFile:  consul.CertFile,
-		KeyFile:   consul.KeyFile,
-		Namespace: consul.Namespace,
+		HTTPAddr:   cc.Addr,
+		Auth:       cc.Auth,
+		SSL:        decodeTriState(cc.EnableSSL),
+		VerifySSL:  decodeTriState(cc.VerifySSL),
+		GRPCCAFile: cc.GRPCCAFile,
+		CAFile:     cc.CAFile,
+		CertFile:   cc.CertFile,
+		KeyFile:    cc.KeyFile,
+		Namespace:  cc.Namespace,
 	}
 }
 
@@ -125,7 +127,7 @@ type envoyBootstrapHook struct {
 	// envoyBootstrapWaitTime is the total amount of time hook will wait for Consul
 	envoyBootstrapWaitTime time.Duration
 
-	// envoyBootstrapInitialGap is the initial wait gap when retyring
+	// envoyBootstrapInitialGap is the initial wait gap when retrying
 	envoyBoostrapInitialGap time.Duration
 
 	// envoyBootstrapMaxJitter is the maximum amount of jitter applied to retries
@@ -542,29 +544,19 @@ func (e envoyBootstrapArgs) args() []string {
 		"-bootstrap",
 	}
 
-	if v := e.gateway; v != "" {
-		arguments = append(arguments, "-gateway", v)
-	}
-
-	if v := e.siToken; v != "" {
-		arguments = append(arguments, "-token", v)
-	}
-
-	if v := e.consulConfig.CAFile; v != "" {
-		arguments = append(arguments, "-ca-file", v)
-	}
-
-	if v := e.consulConfig.CertFile; v != "" {
-		arguments = append(arguments, "-client-cert", v)
-	}
-
-	if v := e.consulConfig.KeyFile; v != "" {
-		arguments = append(arguments, "-client-key", v)
+	appendIfSet := func(param, value string) {
+		if value != "" {
+			arguments = append(arguments, param, value)
+		}
 	}
 
-	if v := e.namespace; v != "" {
-		arguments = append(arguments, "-namespace", v)
-	}
+	appendIfSet("-gateway", e.gateway)
+	appendIfSet("-token", e.siToken)
+	appendIfSet("-grpc-ca-file", e.consulConfig.GRPCCAFile)
+	appendIfSet("-ca-file", e.consulConfig.CAFile)
+	appendIfSet("-client-cert", e.consulConfig.CertFile)
+	appendIfSet("-client-key", e.consulConfig.KeyFile)
+	appendIfSet("-namespace", e.namespace)
 
 	return arguments
 }

client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go

@@ -105,13 +105,14 @@ var (
 	}
 
 	consulTLSConfig = consulTransportConfig{
-		HTTPAddr:  "2.2.2.2",            // arg
-		Auth:      "user:password",      // env
-		SSL:       "true",               // env
-		VerifySSL: "true",               // env
-		CAFile:    "/etc/tls/ca-file",   // arg
-		CertFile:  "/etc/tls/cert-file", // arg
-		KeyFile:   "/etc/tls/key-file",  // arg
+		HTTPAddr:   "2.2.2.2",               // arg
+		Auth:       "user:password",         // env
+		SSL:        "true",                  // env
+		VerifySSL:  "true",                  // env
+		GRPCCAFile: "/etc/tls/grpc-ca-file", // arg
+		CAFile:     "/etc/tls/ca-file",      // arg
+		CertFile:   "/etc/tls/cert-file",    // arg
+		KeyFile:    "/etc/tls/key-file",     // arg
 	}
 )
 
@@ -175,6 +176,7 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 			"-address", "127.0.0.1:19100",
 			"-proxy-id", "s1-sidecar-proxy",
 			"-bootstrap",
+			"-grpc-ca-file", "/etc/tls/grpc-ca-file",
 			"-ca-file", "/etc/tls/ca-file",
 			"-client-cert", "/etc/tls/cert-file",
 			"-client-key", "/etc/tls/key-file",

nomad/structs/config/consul.go

@@ -102,6 +102,11 @@ type ConsulConfig struct {
 	// Uses Consul's default and env var.
 	VerifySSL *bool `hcl:"verify_ssl"`
 
+	// GRPCCAFile is the path to the ca certificate used for Consul gRPC communication.
+	//
+	// Uses Consul's default and env var.
+	GRPCCAFile string `hcl:"grpc_ca_file"`
+
 	// CAFile is the path to the ca certificate used for Consul communication.
 	//
 	// Uses Consul's default and env var.
@@ -219,6 +224,9 @@ func (c *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig {
 	if b.ShareSSL != nil {
 		result.ShareSSL = pointer.Of(*b.ShareSSL)
 	}
+	if b.GRPCCAFile != "" {
+		result.GRPCCAFile = b.GRPCCAFile
+	}
 	if b.CAFile != "" {
 		result.CAFile = b.CAFile
 	}

nomad/structs/config/consul_test.go

@@ -54,6 +54,7 @@ func TestConsulConfig_Merge(t *testing.T) {
 		Auth:                 "1",
 		EnableSSL:            &no,
 		VerifySSL:            &no,
+		GRPCCAFile:           "1",
 		CAFile:               "1",
 		CertFile:             "1",
 		KeyFile:              "1",
@@ -81,6 +82,7 @@ func TestConsulConfig_Merge(t *testing.T) {
 		Auth:                 "2",
 		EnableSSL:            &yes,
 		VerifySSL:            &yes,
+		GRPCCAFile:           "2",
 		CAFile:               "2",
 		CertFile:             "2",
 		KeyFile:              "2",
@@ -108,6 +110,7 @@ func TestConsulConfig_Merge(t *testing.T) {
 		Auth:                 "2",
 		EnableSSL:            &yes,
 		VerifySSL:            &yes,
+		GRPCCAFile:           "2",
 		CAFile:               "2",
 		CertFile:             "2",
 		KeyFile:              "2",

website/content/docs/configuration/consul.mdx

@@ -61,6 +61,10 @@ configuring Nomad to talk to Consul via DNS such as consul.service.consul
   respective services, each tagged appropriately with either `http` or `rpc`
   tag. Nomad servers also advertise a `serf` tagged service.
 
+- `grpc_ca_file` `(string: "")` - Specifies an optional path to the GRPC CA
+   certificate used for communication between Connect sidecar proxies and Consul
+   agents. Will default to the `CONSUL_GRPC_CACERT` environment variable if set.
+
 - `ca_file` `(string: "")` - Specifies an optional path to the CA certificate
   used for Consul communication. This defaults to the system bundle if
   unspecified. Will default to the `CONSUL_CACERT` environment variable if set.

website/content/docs/integrations/consul-connect.mdx

@@ -96,6 +96,27 @@ For JSON configurations:
 }
 ```
 
+#### Consul TLS
+
+~> **Note:** Consul 1.14+ made a [backwards incompatible change][consul_grpc_tls]
+in how TLS enabled grpc listeners work. When using Consul 1.14 with TLS enabled users
+will need to specify additional Nomad agent configuration to work with Connect. The
+`consul.grpc_ca_file` value must now be configured (introduced in Nomad 1.4.4),
+and `consul.grpc_address` will most likely need to be set to use the new standard
+`grpc_tls` port of `8503`.
+
+```hcl
+consul {
+  grpc_ca_file = "/etc/tls/consul-agent-ca.pem"
+  grpc_address = "127.0.0.1:8503"
+  ca_file      = "/etc/tls/consul-agent-ca.pem"
+  cert_file    = "/etc/tls/dc1-client-consul-0.pem"
+  key_file     = "/etc/tls/dc1-client-consul-0-key.pem"
+  ssl          = true
+  address      = "127.0.0.1:8501"
+}
+```
+
 #### Consul ACLs
 
 ~> **Note:** Starting in Nomad v1.3.0, Consul Service Identity ACL tokens automatically
@@ -356,3 +377,4 @@ dashes (`-`) are converted to underscores (`_`) in environment variables so
 [`Local`]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#token-attributes
 [anon_token]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#special-purpose-tokens
 [consul_ports]: https://developer.hashicorp.com/consul/docs/agent/config/config-files#ports
+[consul_grpc_tls]: https://developer.hashicorp.com/consul/docs/upgrading/upgrade-specific#changes-to-grpc-tls-configuration