Inline API Gateway TLS cert code (#16295)

* Include secret type when building resources from config snapshot

* First pass at generating envoy secrets from api-gateway snapshot

* Update comments for xDS update order

* Add secret type + corresponding golden files to existing tests

* Initialize test helpers for testing api-gateway resource generation

* Generate golden files for new api-gateway xDS resource test

* Support ADS for TLS certificates on api-gateway

* Configure TLS on api-gateway listeners

* Inline TLS cert code

* update tests

* Add SNI support so we can have multiple certificates

* Remove commented out section from helper

* regen deep-copy

* Add tcp tls test

---------

Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com>

agent/proxycfg/proxycfg.deepcopy.go

@@ -310,6 +310,23 @@ func (o *configSnapshotAPIGateway) DeepCopy() *configSnapshotAPIGateway {
 			cp.Listeners[k2] = cp_Listeners_v2
 		}
 	}
+	if o.ListenerCertificates != nil {
+		cp.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry, len(o.ListenerCertificates))
+		for k2, v2 := range o.ListenerCertificates {
+			var cp_ListenerCertificates_v2 []structs.InlineCertificateConfigEntry
+			if v2 != nil {
+				cp_ListenerCertificates_v2 = make([]structs.InlineCertificateConfigEntry, len(v2))
+				copy(cp_ListenerCertificates_v2, v2)
+				for i3 := range v2 {
+					{
+						retV := v2[i3].DeepCopy()
+						cp_ListenerCertificates_v2[i3] = *retV
+					}
+				}
+			}
+			cp.ListenerCertificates[k2] = cp_ListenerCertificates_v2
+		}
+	}
 	if o.BoundListeners != nil {
 		cp.BoundListeners = make(map[string]structs.BoundAPIGatewayListener, len(o.BoundListeners))
 		for k2, v2 := range o.BoundListeners {

agent/proxycfg/snapshot.go

@@ -734,6 +734,8 @@ type configSnapshotAPIGateway struct {
 	// Listeners is the original listener config from the api-gateway config
 	// entry to save us trying to pass fields through Upstreams
 	Listeners map[string]structs.APIGatewayListener
+	// this acts as an intermediary for inlining certificates
+	ListenerCertificates map[IngressListenerKey][]structs.InlineCertificateConfigEntry
 
 	BoundListeners map[string]structs.BoundAPIGatewayListener
 }
@@ -751,6 +753,9 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI
 	watchedUpstreamEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes)
 	watchedGatewayEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes)
 
+	// reset the cached certificates
+	c.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry)
+
 	for name, listener := range c.Listeners {
 		boundListener, ok := c.BoundListeners[name]
 		if !ok {
@@ -802,17 +807,18 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI
 			watchedGatewayEndpoints[id] = gatewayEndpoints
 		}
 
+		key := IngressListenerKey{
+			Port:     listener.Port,
+			Protocol: string(listener.Protocol),
+		}
+
 		// Configure TLS for the ingress listener
-		tls, err := c.toIngressTLS()
+		tls, err := c.toIngressTLS(key, listener, boundListener)
 		if err != nil {
 			return configSnapshotIngressGateway{}, err
 		}
-		ingressListener.TLS = tls
 
-		key := IngressListenerKey{
-			Port:     listener.Port,
-			Protocol: string(listener.Protocol),
-		}
+		ingressListener.TLS = tls
 		ingressListeners[key] = ingressListener
 		ingressUpstreams[key] = upstreams
 	}
@@ -905,9 +911,25 @@ DOMAIN_LOOP:
 	return services, upstreams, compiled, err
 }
 
-func (c *configSnapshotAPIGateway) toIngressTLS() (*structs.GatewayTLSConfig, error) {
-	// TODO (t-eckert) this is dependent on future SDS work.
-	return &structs.GatewayTLSConfig{}, nil
+func (c *configSnapshotAPIGateway) toIngressTLS(key IngressListenerKey, listener structs.APIGatewayListener, bound structs.BoundAPIGatewayListener) (*structs.GatewayTLSConfig, error) {
+	if len(listener.TLS.Certificates) == 0 {
+		return nil, nil
+	}
+
+	for _, certRef := range bound.Certificates {
+		cert, ok := c.Certificates.Get(certRef)
+		if !ok {
+			continue
+		}
+		c.ListenerCertificates[key] = append(c.ListenerCertificates[key], *cert)
+	}
+
+	return &structs.GatewayTLSConfig{
+		Enabled:       true,
+		TLSMinVersion: listener.TLS.MinVersion,
+		TLSMaxVersion: listener.TLS.MaxVersion,
+		CipherSuites:  listener.TLS.CipherSuites,
+	}, nil
 }
 
 type configSnapshotIngressGateway struct {

agent/proxycfg/testing_api_gateway.go

@@ -0,0 +1,157 @@
+package proxycfg
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+func TestConfigSnapshotAPIGateway(
+	t testing.T,
+	variation string,
+	nsFn func(ns *structs.NodeService),
+	configFn func(entry *structs.APIGatewayConfigEntry, boundEntry *structs.BoundAPIGatewayConfigEntry),
+	routes []structs.BoundRoute,
+	certificates []structs.InlineCertificateConfigEntry,
+	extraUpdates []UpdateEvent,
+	additionalEntries ...structs.ConfigEntry,
+) *ConfigSnapshot {
+	roots, placeholderLeaf := TestCerts(t)
+
+	entry := &structs.APIGatewayConfigEntry{
+		Kind: structs.APIGateway,
+		Name: "api-gateway",
+	}
+	boundEntry := &structs.BoundAPIGatewayConfigEntry{
+		Kind: structs.BoundAPIGateway,
+		Name: "api-gateway",
+	}
+
+	if configFn != nil {
+		configFn(entry, boundEntry)
+	}
+
+	baseEvents := []UpdateEvent{
+		{
+			CorrelationID: rootsWatchID,
+			Result:        roots,
+		},
+		{
+			CorrelationID: leafWatchID,
+			Result:        placeholderLeaf,
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: entry,
+			},
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: boundEntry,
+			},
+		},
+	}
+
+	for _, route := range routes {
+		// Add the watch event for the route.
+		watch := UpdateEvent{
+			CorrelationID: routeConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: route,
+			},
+		}
+		baseEvents = append(baseEvents, watch)
+
+		// Add the watch event for the discovery chain.
+		entries := []structs.ConfigEntry{
+			&structs.ProxyConfigEntry{
+				Kind: structs.ProxyDefaults,
+				Name: structs.ProxyConfigGlobal,
+				Config: map[string]interface{}{
+					"protocol": route.GetProtocol(),
+				},
+			},
+			&structs.ServiceResolverConfigEntry{
+				Kind: structs.ServiceResolver,
+				Name: "api-gateway",
+			},
+		}
+
+		// Add a discovery chain watch event for each service.
+		for _, serviceName := range route.GetServiceNames() {
+			discoChain := UpdateEvent{
+				CorrelationID: fmt.Sprintf("discovery-chain:%s", UpstreamIDString("", "", serviceName.Name, &serviceName.EnterpriseMeta, "")),
+				Result: &structs.DiscoveryChainResponse{
+					Chain: discoverychain.TestCompileConfigEntries(t, serviceName.Name, "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...),
+				},
+			}
+			baseEvents = append(baseEvents, discoChain)
+		}
+	}
+
+	for _, certificate := range certificates {
+		inlineCertificate := certificate
+		baseEvents = append(baseEvents, UpdateEvent{
+			CorrelationID: inlineCertificateConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &inlineCertificate,
+			},
+		})
+	}
+
+	upstreams := structs.TestUpstreams(t)
+
+	baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot(
+		t, variation, upstreams, additionalEntries...,
+	))
+
+	return testConfigSnapshotFixture(t, &structs.NodeService{
+		Kind:            structs.ServiceKindAPIGateway,
+		Service:         "api-gateway",
+		Address:         "1.2.3.4",
+		Meta:            nil,
+		TaggedAddresses: nil,
+	}, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates))
+}
+
+// TestConfigSnapshotAPIGateway_NilConfigEntry is used to test when
+// the update event for the config entry returns nil
+// since this always happens on the first watch if it doesn't exist.
+func TestConfigSnapshotAPIGateway_NilConfigEntry(
+	t testing.T,
+) *ConfigSnapshot {
+	roots, _ := TestCerts(t)
+
+	baseEvents := []UpdateEvent{
+		{
+			CorrelationID: rootsWatchID,
+			Result:        roots,
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist.
+			},
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist.
+			},
+		},
+	}
+
+	return testConfigSnapshotFixture(t, &structs.NodeService{
+		Kind:            structs.ServiceKindAPIGateway,
+		Service:         "api-gateway",
+		Address:         "1.2.3.4",
+		Meta:            nil,
+		TaggedAddresses: nil,
+	}, nil, nil, testSpliceEvents(baseEvents, nil))
+}

agent/structs/config_entry_inline_certificate.go

@@ -64,6 +64,30 @@ func (e *InlineCertificateConfigEntry) Validate() error {
 	return nil
 }
 
+func (e *InlineCertificateConfigEntry) Hosts() ([]string, error) {
+	certificateBlock, _ := pem.Decode([]byte(e.Certificate))
+	if certificateBlock == nil {
+		return nil, errors.New("failed to parse certificate PEM")
+	}
+
+	certificate, err := x509.ParseCertificate(certificateBlock.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse certificate: %w", err)
+	}
+
+	hosts := []string{certificate.Subject.CommonName}
+
+	for _, name := range certificate.DNSNames {
+		hosts = append(hosts, name)
+	}
+
+	for _, ip := range certificate.IPAddresses {
+		hosts = append(hosts, ip.String())
+	}
+
+	return hosts, nil
+}
+
 func (e *InlineCertificateConfigEntry) CanRead(authz acl.Authorizer) error {
 	var authzContext acl.AuthorizerContext
 	e.FillAuthzContext(&authzContext)

agent/structs/config_entry_status.go

@@ -1,6 +1,7 @@
 package structs
 
 import (
+	"fmt"
 	"sort"
 	"time"
 
@@ -24,6 +25,10 @@ type ResourceReference struct {
 	acl.EnterpriseMeta
 }
 
+func (r *ResourceReference) String() string {
+	return fmt.Sprintf("%s:%s/%s/%s/%s", r.Kind, r.PartitionOrDefault(), r.NamespaceOrDefault(), r.Name, r.SectionName)
+}
+
 func (r *ResourceReference) IsSame(other *ResourceReference) bool {
 	if r == nil && other == nil {
 		return true

agent/xds/delta.go

@@ -466,20 +466,21 @@ func (s *Server) applyEnvoyExtensions(resources *xdscommon.IndexedResources, cfg
 	return nil
 }
 
+// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#eventual-consistency-considerations
 var xDSUpdateOrder = []xDSUpdateOperation{
-	// TODO Update comments
+	// 1. SDS updates (if any) can be pushed here with no harm.
 	{TypeUrl: xdscommon.SecretType, Upsert: true},
-	// 1. CDS updates (if any) must always be pushed first.
+	// 2. CDS updates (if any) must always be pushed before the following types.
 	{TypeUrl: xdscommon.ClusterType, Upsert: true},
-	// 2. EDS updates (if any) must arrive after CDS updates for the respective clusters.
+	// 3. EDS updates (if any) must arrive after CDS updates for the respective clusters.
 	{TypeUrl: xdscommon.EndpointType, Upsert: true},
-	// 3. LDS updates must arrive after corresponding CDS/EDS updates.
+	// 4. LDS updates must arrive after corresponding CDS/EDS updates.
 	{TypeUrl: xdscommon.ListenerType, Upsert: true, Remove: true},
-	// 4. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates.
+	// 5. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates.
 	{TypeUrl: xdscommon.RouteType, Upsert: true, Remove: true},
-	// 5. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates.
+	// 6. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates.
 	// {},
-	// 6. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed.
+	// 7. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed.
 	{TypeUrl: xdscommon.ClusterType, Remove: true},
 	{TypeUrl: xdscommon.EndpointType, Remove: true},
 	{TypeUrl: xdscommon.SecretType, Remove: true},

agent/xds/listeners.go

@@ -2328,6 +2328,9 @@ func makeHTTPInspectorListenerFilter() (*envoy_listener_v3.ListenerFilter, error
 }
 
 func makeSNIFilterChainMatch(sniMatches ...string) *envoy_listener_v3.FilterChainMatch {
+	if sniMatches == nil {
+		return nil
+	}
 	return &envoy_listener_v3.FilterChainMatch{
 		ServerNames: sniMatches,
 	}