Merge pull request #9829 from hashicorp/f-terminating-gateway

consul/connect: Add support for Connect terminating gateways

CHANGELOG.md

@@ -1,10 +1,12 @@
 ## 1.0.3 (Unreleased)
 
+FEATURES:
+ * **Terminating Gateways**: Adds built-in support for running Consul Connect terminating gateways [[GH-9829](https://github.com/hashicorp/nomad/pull/9829)]
+
 IMPROVEMENTS:
  * consul/connect: Made handling of sidecar task container image URLs consistent with the `docker` task driver. [[GH-9580](https://github.com/hashicorp/nomad/issues/9580)]
 
 BUG FIXES:
-
  * consul: Fixed a bug where failing tasks with group services would only cause the allocation to restart once instead of respecting the `restart` field. [[GH-9869](https://github.com/hashicorp/nomad/issues/9869)]
  * consul/connect: Fixed a bug where gateway proxy connection default timeout not set [[GH-9851](https://github.com/hashicorp/nomad/pull/9851)]
  * consul/connect: Fixed a bug preventing more than one connect gateway per Nomad client [[GH-9849](https://github.com/hashicorp/nomad/pull/9849)]

api/services.go

@@ -302,8 +302,8 @@ type ConsulGateway struct {
 	// Ingress represents the Consul Configuration Entry for an Ingress Gateway.
 	Ingress *ConsulIngressConfigEntry `hcl:"ingress,block"`
 
-	// Terminating is not yet supported.
-	// Terminating *ConsulTerminatingConfigEntry
+	// Terminating represents the Consul Configuration Entry for a Terminating Gateway.
+	Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"`
 
 	// Mesh is not yet supported.
 	// Mesh *ConsulMeshConfigEntry
@@ -315,6 +315,7 @@ func (g *ConsulGateway) Canonicalize() {
 	}
 	g.Proxy.Canonicalize()
 	g.Ingress.Canonicalize()
+	g.Terminating.Canonicalize()
 }
 
 func (g *ConsulGateway) Copy() *ConsulGateway {
@@ -323,8 +324,9 @@ func (g *ConsulGateway) Copy() *ConsulGateway {
 	}
 
 	return &ConsulGateway{
-		Proxy:   g.Proxy.Copy(),
-		Ingress: g.Ingress.Copy(),
+		Proxy:       g.Proxy.Copy(),
+		Ingress:     g.Ingress.Copy(),
+		Terminating: g.Terminating.Copy(),
 	}
 }
 
@@ -335,8 +337,8 @@ type ConsulGatewayBindAddress struct {
 }
 
 var (
-	// defaultConnectTimeout is the default amount of time a connect gateway will
-	// wait for a response from an upstream service (same as consul)
+	// defaultGatewayConnectTimeout is the default amount of time connections to
+	// upstreams are allowed before timing out.
 	defaultGatewayConnectTimeout = 5 * time.Second
 )
 
@@ -349,6 +351,7 @@ type ConsulGatewayProxy struct {
 	EnvoyGatewayBindTaggedAddresses bool                                 `mapstructure:"envoy_gateway_bind_tagged_addresses" hcl:"envoy_gateway_bind_tagged_addresses,optional"`
 	EnvoyGatewayBindAddresses       map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses" hcl:"envoy_gateway_bind_addresses,block"`
 	EnvoyGatewayNoDefaultBind       bool                                 `mapstructure:"envoy_gateway_no_default_bind" hcl:"envoy_gateway_no_default_bind,optional"`
+	EnvoyDNSDiscoveryType           string                               `mapstructure:"envoy_dns_discovery_type" hcl:"envoy_dns_discovery_type,optional"`
 	Config                          map[string]interface{}               `hcl:"config,block"` // escape hatch envoy config
 }
 
@@ -397,6 +400,7 @@ func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy {
 		EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses,
 		EnvoyGatewayBindAddresses:       binds,
 		EnvoyGatewayNoDefaultBind:       p.EnvoyGatewayNoDefaultBind,
+		EnvoyDNSDiscoveryType:           p.EnvoyDNSDiscoveryType,
 		Config:                          config,
 	}
 }
@@ -549,9 +553,74 @@ func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry {
 	}
 }
 
-// ConsulTerminatingConfigEntry is not yet supported.
-// type ConsulTerminatingConfigEntry struct {
-// }
+type ConsulLinkedService struct {
+	Name     string `hcl:"name,optional"`
+	CAFile   string `hcl:"ca_file,optional"`
+	CertFile string `hcl:"cert_file,optional"`
+	KeyFile  string `hcl:"key_file,optional"`
+	SNI      string `hcl:"sni,optional"`
+}
+
+func (s *ConsulLinkedService) Canonicalize() {
+	// nothing to do for now
+}
+
+func (s *ConsulLinkedService) Copy() *ConsulLinkedService {
+	if s == nil {
+		return nil
+	}
+
+	return &ConsulLinkedService{
+		Name:     s.Name,
+		CAFile:   s.CAFile,
+		CertFile: s.CertFile,
+		KeyFile:  s.KeyFile,
+		SNI:      s.SNI,
+	}
+}
+
+// ConsulTerminatingConfigEntry represents the Consul Configuration Entry type
+// for a Terminating Gateway.
+//
+// https://www.consul.io/docs/agent/config-entries/terminating-gateway#available-fields
+type ConsulTerminatingConfigEntry struct {
+	// Namespace is not yet supported.
+	// Namespace string
+
+	Services []*ConsulLinkedService `hcl:"service,block"`
+}
+
+func (e *ConsulTerminatingConfigEntry) Canonicalize() {
+	if e == nil {
+		return
+	}
+
+	if len(e.Services) == 0 {
+		e.Services = nil
+	}
+
+	for _, service := range e.Services {
+		service.Canonicalize()
+	}
+}
+
+func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry {
+	if e == nil {
+		return nil
+	}
+
+	var services []*ConsulLinkedService = nil
+	if n := len(e.Services); n > 0 {
+		services = make([]*ConsulLinkedService, n)
+		for i := 0; i < n; i++ {
+			services[i] = e.Services[i].Copy()
+		}
+	}
+
+	return &ConsulTerminatingConfigEntry{
+		Services: services,
+	}
+}
 
 // ConsulMeshConfigEntry is not yet supported.
 // type ConsulMeshConfigEntry struct {

api/services_test.go

@@ -291,7 +291,10 @@ func TestService_ConsulGateway_Canonicalize(t *testing.T) {
 		}
 		cg.Canonicalize()
 		require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout)
+		require.True(t, cg.Proxy.EnvoyGatewayBindTaggedAddresses)
 		require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses)
+		require.True(t, cg.Proxy.EnvoyGatewayNoDefaultBind)
+		require.Empty(t, cg.Proxy.EnvoyDNSDiscoveryType)
 		require.Nil(t, cg.Proxy.Config)
 		require.Nil(t, cg.Ingress.Listeners)
 	})
@@ -314,6 +317,7 @@ func TestService_ConsulGateway_Copy(t *testing.T) {
 				"listener2": {Address: "10.0.0.1", Port: 2001},
 			},
 			EnvoyGatewayNoDefaultBind: true,
+			EnvoyDNSDiscoveryType:     "STRICT_DNS",
 			Config: map[string]interface{}{
 				"foo": "bar",
 				"baz": 3,
@@ -334,6 +338,11 @@ func TestService_ConsulGateway_Copy(t *testing.T) {
 				}},
 			},
 		},
+		Terminating: &ConsulTerminatingConfigEntry{
+			Services: []*ConsulLinkedService{{
+				Name: "linked-service1",
+			}},
+		},
 	}
 
 	t.Run("complete", func(t *testing.T) {
@@ -418,3 +427,47 @@ func TestService_ConsulIngressConfigEntry_Copy(t *testing.T) {
 		require.Equal(t, entry, result)
 	})
 }
+
+func TestService_ConsulTerminatingConfigEntry_Canonicalize(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		c := (*ConsulTerminatingConfigEntry)(nil)
+		c.Canonicalize()
+		require.Nil(t, c)
+	})
+
+	t.Run("empty services", func(t *testing.T) {
+		c := &ConsulTerminatingConfigEntry{
+			Services: []*ConsulLinkedService{},
+		}
+		c.Canonicalize()
+		require.Nil(t, c.Services)
+	})
+}
+
+func TestService_ConsulTerminatingConfigEntry_Copy(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		result := (*ConsulIngressConfigEntry)(nil).Copy()
+		require.Nil(t, result)
+	})
+
+	entry := &ConsulTerminatingConfigEntry{
+		Services: []*ConsulLinkedService{{
+			Name: "servic1",
+		}, {
+			Name:     "service2",
+			CAFile:   "ca_file.pem",
+			CertFile: "cert_file.pem",
+			KeyFile:  "key_file.pem",
+			SNI:      "sni.terminating.consul",
+		}},
+	}
+
+	t.Run("complete", func(t *testing.T) {
+		result := entry.Copy()
+		require.Equal(t, entry, result)
+	})
+}

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -110,13 +110,16 @@ func (envoyBootstrapHook) Name() string {
 	return envoyBootstrapHookName
 }
 
+func isConnectKind(kind string) bool {
+	kinds := []string{structs.ConnectProxyPrefix, structs.ConnectIngressPrefix, structs.ConnectTerminatingPrefix}
+	return helper.SliceStringContains(kinds, kind)
+}
+
 func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string, string, error) {
-	serviceKind := kind.Name()
 	serviceName := kind.Value()
+	serviceKind := kind.Name()
 
-	switch serviceKind {
-	case structs.ConnectProxyPrefix, structs.ConnectIngressPrefix:
-	default:
+	if !isConnectKind(serviceKind) {
 		return "", "", errors.New("envoy must be used as connect sidecar or gateway")
 	}
 
@@ -350,13 +353,15 @@ func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
 		proxyID      string // gateway only
 	)
 
-	if service.Connect.HasSidecar() {
+	switch {
+	case service.Connect.HasSidecar():
 		sidecarForID = h.proxyServiceID(group, service)
-	}
-
-	if service.Connect.IsGateway() {
-		gateway = "ingress" // more types in the future
+	case service.Connect.IsIngress():
+		proxyID = h.proxyServiceID(group, service)
+		gateway = "ingress"
+	case service.Connect.IsTerminating():
 		proxyID = h.proxyServiceID(group, service)
+		gateway = "terminating"
 	}
 
 	h.logger.Debug("bootstrapping envoy",

command/agent/consul/connect.go

@@ -26,13 +26,15 @@ func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.
 		return &api.AgentServiceConnect{Native: true}, nil
 
 	case nc.HasSidecar():
+		// must register the sidecar for this service
 		sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
 		if err != nil {
 			return nil, err
 		}
 		return &api.AgentServiceConnect{SidecarService: sidecarReg}, nil
 
 	default:
+		// a non-nil but empty connect block makes no sense
 		return nil, fmt.Errorf("Connect configuration empty for service %s", serviceName)
 	}
 }
@@ -64,6 +66,10 @@ func newConnectGateway(serviceName string, connect *structs.ConsulConnect) *api.
 			envoyConfig["envoy_gateway_bind_tagged_addresses"] = true
 		}
 
+		if proxy.EnvoyDNSDiscoveryType != "" {
+			envoyConfig["envoy_dns_discovery_type"] = proxy.EnvoyDNSDiscoveryType
+		}
+
 		if proxy.ConnectTimeout != nil {
 			envoyConfig["connect_timeout_ms"] = proxy.ConnectTimeout.Milliseconds()
 		}
@@ -89,7 +95,7 @@ func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarSe
 		return nil, err
 	}
 
-	proxy, err := connectProxy(css.Proxy, cPort.To, networks)
+	proxy, err := connectSidecarProxy(css.Proxy, cPort.To, networks)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +108,7 @@ func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarSe
 	}, nil
 }
 
-func connectProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
+func connectSidecarProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
 	if proxy == nil {
 		proxy = new(structs.ConsulProxy)
 	}

command/agent/consul/connect_test.go

@@ -119,7 +119,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	// If the input proxy is nil, we expect the output to be a proxy with its
 	// config set to default values.
 	t.Run("nil proxy", func(t *testing.T) {
-		proxy, err := connectProxy(nil, 2000, testConnectNetwork)
+		proxy, err := connectSidecarProxy(nil, 2000, testConnectNetwork)
 		require.NoError(t, err)
 		require.Equal(t, &api.AgentServiceConnectProxyConfig{
 			LocalServiceAddress: "",
@@ -134,7 +134,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	})
 
 	t.Run("bad proxy", func(t *testing.T) {
-		_, err := connectProxy(&structs.ConsulProxy{
+		_, err := connectSidecarProxy(&structs.ConsulProxy{
 			LocalServiceAddress: "0.0.0.0",
 			LocalServicePort:    2000,
 			Upstreams:           nil,
@@ -149,7 +149,7 @@ func TestConnect_connectProxy(t *testing.T) {
 	})
 
 	t.Run("normal", func(t *testing.T) {
-		proxy, err := connectProxy(&structs.ConsulProxy{
+		proxy, err := connectSidecarProxy(&structs.ConsulProxy{
 			LocalServiceAddress: "0.0.0.0",
 			LocalServicePort:    2000,
 			Upstreams:           nil,
@@ -453,6 +453,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
 						},
 					},
 					EnvoyGatewayNoDefaultBind: true,
+					EnvoyDNSDiscoveryType:     "STRICT_DNS",
 					Config: map[string]interface{}{
 						"foo": 1,
 					},
@@ -470,6 +471,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
 					},
 				},
 				"envoy_gateway_no_default_bind": true,
+				"envoy_dns_discovery_type":      "STRICT_DNS",
 				"foo":                           1,
 			},
 		}, result)

command/agent/consul/service_client.go

@@ -891,10 +891,21 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
 	// This enables the consul UI to show that Nomad registered this service
 	meta["external-source"] = "nomad"
 
-	// Explicitly set the service kind in case this service represents a Connect gateway.
+	// Explicitly set the Consul service Kind in case this service represents
+	// one of the Connect gateway types.
 	kind := api.ServiceKindTypical
-	if service.Connect.IsGateway() {
+	switch {
+	case service.Connect.IsIngress():
 		kind = api.ServiceKindIngressGateway
+	case service.Connect.IsTerminating():
+		kind = api.ServiceKindTerminatingGateway
+		// set the default port if bridge / default listener set
+		if defaultBind, exists := service.Connect.Gateway.Proxy.EnvoyGatewayBindAddresses["default"]; exists {
+			portLabel := fmt.Sprintf("%s-%s", structs.ConnectTerminatingPrefix, service.Name)
+			if dynPort, ok := workload.Ports.Get(portLabel); ok {
+				defaultBind.Port = dynPort.Value
+			}
+		}
 	}
 
 	// Build the Consul Service registration request