[NET-8091] Add file-system-certificate config entry for API gateway

.changelog/20873.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+gateways: api-gateway can leverage listener TLS certificates available on the gateway's local filesystem by specifying the public certificate and private key path in the new file-system-certificate configuration entry
+```

agent/consul/fsm/decode_downgrade.go

@@ -619,6 +619,8 @@ func MakeShadowConfigEntry(kind, name string) (structs.ConfigEntry, error) {
 		return &ShadowAPIGatewayConfigEntry{APIGatewayConfigEntry: &structs.APIGatewayConfigEntry{Name: name}}, nil
 	case structs.BoundAPIGateway:
 		return &ShadowBoundAPIGatewayConfigEntry{BoundAPIGatewayConfigEntry: &structs.BoundAPIGatewayConfigEntry{Name: name}}, nil
+	case structs.FileSystemCertificate:
+		return &ShadowFileSystemCertificateConfigEntry{FileSystemCertificateConfigEntry: &structs.FileSystemCertificateConfigEntry{Name: name}}, nil
 	case structs.InlineCertificate:
 		return &ShadowInlineCertificateConfigEntry{InlineCertificateConfigEntry: &structs.InlineCertificateConfigEntry{Name: name}}, nil
 	case structs.HTTPRoute:
@@ -931,6 +933,15 @@ func (s ShadowBoundAPIGatewayConfigEntry) GetRealConfigEntry() structs.ConfigEnt
 	return s.BoundAPIGatewayConfigEntry
 }
 
+type ShadowFileSystemCertificateConfigEntry struct {
+	ShadowBase
+	*structs.FileSystemCertificateConfigEntry
+}
+
+func (s ShadowFileSystemCertificateConfigEntry) GetRealConfigEntry() structs.ConfigEntry {
+	return s.FileSystemCertificateConfigEntry
+}
+
 type ShadowInlineCertificateConfigEntry struct {
 	ShadowBase
 	*structs.InlineCertificateConfigEntry

agent/consul/fsm/fsm.go

@@ -403,6 +403,11 @@ func (c *FSM) registerStreamSnapshotHandlers() {
 	}, true)
 	panicIfErr(err)
 
+	err = c.deps.Publisher.RegisterHandler(state.EventTopicFileSystemCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
+		return c.State().FileSystemCertificateSnapshot(req, buf)
+	}, true)
+	panicIfErr(err)
+
 	err = c.deps.Publisher.RegisterHandler(state.EventTopicInlineCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
 		return c.State().InlineCertificateSnapshot(req, buf)
 	}, true)

agent/consul/gateways/controller_gateways.go

@@ -61,7 +61,7 @@ func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Req
 		return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileHTTPRoute, r.cleanupRoute)
 	case structs.TCPRoute:
 		return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileTCPRoute, r.cleanupRoute)
-	case structs.InlineCertificate:
+	case structs.InlineCertificate, structs.FileSystemCertificate:
 		return r.enqueueCertificateReferencedGateways(r.fsm.State(), ctx, req)
 	case structs.JWTProvider:
 		return r.enqueueJWTProviderReferencedGatewaysAndHTTPRoutes(r.fsm.State(), ctx, req)

agent/consul/state/config_entry.go

@@ -646,6 +646,7 @@ func validateProposedConfigEntryInGraph(
 	case structs.ExportedServices:
 	case structs.APIGateway: // TODO Consider checkGatewayClash
 	case structs.BoundAPIGateway:
+	case structs.FileSystemCertificate:
 	case structs.InlineCertificate:
 	case structs.HTTPRoute:
 	case structs.TCPRoute:

agent/consul/state/config_entry_events.go

@@ -15,20 +15,21 @@ import (
 
 // Adding events for a new config entry kind? Remember to update ConfigEntryFromStructs and ConfigEntryToStructs.
 var configEntryKindToTopic = map[string]stream.Topic{
-	structs.MeshConfig:        EventTopicMeshConfig,
-	structs.ServiceResolver:   EventTopicServiceResolver,
-	structs.IngressGateway:    EventTopicIngressGateway,
-	structs.ServiceIntentions: EventTopicServiceIntentions,
-	structs.ServiceDefaults:   EventTopicServiceDefaults,
-	structs.APIGateway:        EventTopicAPIGateway,
-	structs.TCPRoute:          EventTopicTCPRoute,
-	structs.HTTPRoute:         EventTopicHTTPRoute,
-	structs.InlineCertificate: EventTopicInlineCertificate,
-	structs.BoundAPIGateway:   EventTopicBoundAPIGateway,
-	structs.RateLimitIPConfig: EventTopicIPRateLimit,
-	structs.SamenessGroup:     EventTopicSamenessGroup,
-	structs.JWTProvider:       EventTopicJWTProvider,
-	structs.ExportedServices:  EventTopicExportedServices,
+	structs.MeshConfig:            EventTopicMeshConfig,
+	structs.ServiceResolver:       EventTopicServiceResolver,
+	structs.IngressGateway:        EventTopicIngressGateway,
+	structs.ServiceIntentions:     EventTopicServiceIntentions,
+	structs.ServiceDefaults:       EventTopicServiceDefaults,
+	structs.APIGateway:            EventTopicAPIGateway,
+	structs.TCPRoute:              EventTopicTCPRoute,
+	structs.HTTPRoute:             EventTopicHTTPRoute,
+	structs.FileSystemCertificate: EventTopicFileSystemCertificate,
+	structs.InlineCertificate:     EventTopicInlineCertificate,
+	structs.BoundAPIGateway:       EventTopicBoundAPIGateway,
+	structs.RateLimitIPConfig:     EventTopicIPRateLimit,
+	structs.SamenessGroup:         EventTopicSamenessGroup,
+	structs.JWTProvider:           EventTopicJWTProvider,
+	structs.ExportedServices:      EventTopicExportedServices,
 }
 
 // EventSubjectConfigEntry is a stream.Subject used to route and receive events
@@ -147,6 +148,12 @@ func (s *Store) HTTPRouteSnapshot(req stream.SubscribeRequest, buf stream.Snapsh
 	return s.configEntrySnapshot(structs.HTTPRoute, req, buf)
 }
 
+// FileSystemCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
+// inline-certificate config entries.
+func (s *Store) FileSystemCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
+	return s.configEntrySnapshot(structs.FileSystemCertificate, req, buf)
+}
+
 // InlineCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
 // inline-certificate config entries.
 func (s *Store) InlineCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {

agent/consul/state/events.go

@@ -46,7 +46,8 @@ func PBToStreamSubscribeRequest(req *pbsubscribe.SubscribeRequest, entMeta acl.E
 		case EventTopicMeshConfig, EventTopicServiceResolver, EventTopicIngressGateway,
 			EventTopicServiceIntentions, EventTopicServiceDefaults, EventTopicAPIGateway,
 			EventTopicTCPRoute, EventTopicHTTPRoute, EventTopicJWTProvider, EventTopicInlineCertificate,
-			EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices:
+			EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices,
+			EventTopicFileSystemCertificate:
 			subject = EventSubjectConfigEntry{
 				Name:           named.Key,
 				EnterpriseMeta: &entMeta,

agent/consul/state/memdb.go

@@ -196,23 +196,24 @@ func (db *readDB) ReadTxn() AbortTxn {
 }
 
 var (
-	EventTopicServiceHealth        = pbsubscribe.Topic_ServiceHealth
-	EventTopicServiceHealthConnect = pbsubscribe.Topic_ServiceHealthConnect
-	EventTopicMeshConfig           = pbsubscribe.Topic_MeshConfig
-	EventTopicServiceResolver      = pbsubscribe.Topic_ServiceResolver
-	EventTopicIngressGateway       = pbsubscribe.Topic_IngressGateway
-	EventTopicServiceIntentions    = pbsubscribe.Topic_ServiceIntentions
-	EventTopicServiceDefaults      = pbsubscribe.Topic_ServiceDefaults
-	EventTopicServiceList          = pbsubscribe.Topic_ServiceList
-	EventTopicAPIGateway           = pbsubscribe.Topic_APIGateway
-	EventTopicTCPRoute             = pbsubscribe.Topic_TCPRoute
-	EventTopicHTTPRoute            = pbsubscribe.Topic_HTTPRoute
-	EventTopicInlineCertificate    = pbsubscribe.Topic_InlineCertificate
-	EventTopicBoundAPIGateway      = pbsubscribe.Topic_BoundAPIGateway
-	EventTopicIPRateLimit          = pbsubscribe.Topic_IPRateLimit
-	EventTopicSamenessGroup        = pbsubscribe.Topic_SamenessGroup
-	EventTopicJWTProvider          = pbsubscribe.Topic_JWTProvider
-	EventTopicExportedServices     = pbsubscribe.Topic_ExportedServices
+	EventTopicServiceHealth         = pbsubscribe.Topic_ServiceHealth
+	EventTopicServiceHealthConnect  = pbsubscribe.Topic_ServiceHealthConnect
+	EventTopicMeshConfig            = pbsubscribe.Topic_MeshConfig
+	EventTopicServiceResolver       = pbsubscribe.Topic_ServiceResolver
+	EventTopicIngressGateway        = pbsubscribe.Topic_IngressGateway
+	EventTopicServiceIntentions     = pbsubscribe.Topic_ServiceIntentions
+	EventTopicServiceDefaults       = pbsubscribe.Topic_ServiceDefaults
+	EventTopicServiceList           = pbsubscribe.Topic_ServiceList
+	EventTopicAPIGateway            = pbsubscribe.Topic_APIGateway
+	EventTopicTCPRoute              = pbsubscribe.Topic_TCPRoute
+	EventTopicHTTPRoute             = pbsubscribe.Topic_HTTPRoute
+	EventTopicFileSystemCertificate = pbsubscribe.Topic_FileSystemCertificate
+	EventTopicInlineCertificate     = pbsubscribe.Topic_InlineCertificate
+	EventTopicBoundAPIGateway       = pbsubscribe.Topic_BoundAPIGateway
+	EventTopicIPRateLimit           = pbsubscribe.Topic_IPRateLimit
+	EventTopicSamenessGroup         = pbsubscribe.Topic_SamenessGroup
+	EventTopicJWTProvider           = pbsubscribe.Topic_JWTProvider
+	EventTopicExportedServices      = pbsubscribe.Topic_ExportedServices
 )
 
 func processDBChanges(tx ReadTxn, changes Changes) ([]stream.Event, error) {

agent/consul/usagemetrics/usagemetrics_test.go

@@ -334,6 +334,14 @@ var baseCases = map[string]testCase{
 					{Name: "kind", Value: "bound-api-gateway"},
 				},
 			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "file-system-certificate"},
+				},
+			},
 			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
 				Name:  "consul.usage.test.state.config_entries",
 				Value: 0,
@@ -607,6 +615,14 @@ var baseCases = map[string]testCase{
 					{Name: "kind", Value: "bound-api-gateway"},
 				},
 			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "file-system-certificate"},
+				},
+			},
 			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
 				Name:  "consul.usage.test.state.config_entries",
 				Value: 0,

agent/proxycfg-glue/config_entry.go

@@ -69,6 +69,8 @@ func newConfigEntryRequest(req *structs.ConfigEntryQuery, deps ServerDataSourceD
 		topic = pbsubscribe.Topic_HTTPRoute
 	case structs.TCPRoute:
 		topic = pbsubscribe.Topic_TCPRoute
+	case structs.FileSystemCertificate:
+		topic = pbsubscribe.Topic_FileSystemCertificate
 	case structs.InlineCertificate:
 		topic = pbsubscribe.Topic_InlineCertificate
 	case structs.BoundAPIGateway:

agent/proxycfg/api_gateway.go

@@ -64,6 +64,7 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err
 	snap.APIGateway.HTTPRoutes = watch.NewMap[structs.ResourceReference, *structs.HTTPRouteConfigEntry]()
 	snap.APIGateway.TCPRoutes = watch.NewMap[structs.ResourceReference, *structs.TCPRouteConfigEntry]()
 	snap.APIGateway.Certificates = watch.NewMap[structs.ResourceReference, *structs.InlineCertificateConfigEntry]()
+	snap.APIGateway.FSCertificates = watch.NewMap[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]()
 
 	snap.APIGateway.Upstreams = make(listenerRouteUpstreams)
 	snap.APIGateway.UpstreamsSet = make(routeUpstreamSet)
@@ -96,7 +97,8 @@ func (h *handlerAPIGateway) subscribeToConfigEntry(ctx context.Context, kind, na
 // handleUpdate responds to changes in the api-gateway. In general, we want
 // to crawl the various resources related to or attached to the gateway and
 // collect the list of things need to generate xDS.  This list of resources
-// includes the bound-api-gateway, http-routes, tcp-routes, and inline-certificates.
+// includes the bound-api-gateway, http-routes, tcp-routes,
+// file-system-certificates and inline-certificates.
 func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
 	if u.Err != nil {
 		return fmt.Errorf("error filling agent cache: %v", u.Err)
@@ -113,6 +115,11 @@ func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, sna
 		if err := h.handleGatewayConfigUpdate(ctx, u, snap, u.CorrelationID); err != nil {
 			return err
 		}
+	case fileSystemCertificateConfigWatchID:
+		// Handle change in an attached file-system-certificate config entry
+		if err := h.handleFileSystemCertConfigUpdate(ctx, u, snap); err != nil {
+			return err
+		}
 	case inlineCertificateConfigWatchID:
 		// Handle change in an attached inline-certificate config entry
 		if err := h.handleInlineCertConfigUpdate(ctx, u, snap); err != nil {
@@ -205,12 +212,21 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 			for _, ref := range listener.Certificates {
 				ctx, cancel := context.WithCancel(ctx)
 				seenRefs[ref] = struct{}{}
-				snap.APIGateway.Certificates.InitWatch(ref, cancel)
 
-				err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
-				if err != nil {
-					// TODO May want to continue
-					return err
+				if ref.Kind == structs.FileSystemCertificate {
+					snap.APIGateway.FSCertificates.InitWatch(ref, cancel)
+
+					err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, fileSystemCertificateConfigWatchID)
+					if err != nil {
+						return err
+					}
+				} else {
+					snap.APIGateway.Certificates.InitWatch(ref, cancel)
+
+					err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
@@ -241,6 +257,13 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 			return true
 		})
 
+		snap.APIGateway.FSCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
+			if _, ok := seenRefs[ref]; !ok {
+				snap.APIGateway.FSCertificates.CancelWatch(ref)
+			}
+			return true
+		})
+
 		snap.APIGateway.BoundGatewayConfigLoaded = true
 		break
 	case *structs.APIGatewayConfigEntry:
@@ -265,6 +288,30 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 	return h.watchIngressLeafCert(ctx, snap)
 }
 
+func (h *handlerAPIGateway) handleFileSystemCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
+	resp, ok := u.Result.(*structs.ConfigEntryResponse)
+	if !ok {
+		return fmt.Errorf("invalid type for response: %T", u.Result)
+	} else if resp == nil || resp.Entry == nil {
+		return nil
+	}
+
+	cfg, ok := resp.Entry.(*structs.FileSystemCertificateConfigEntry)
+	if !ok || cfg == nil {
+		return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
+	}
+
+	ref := structs.ResourceReference{
+		Kind:           cfg.GetKind(),
+		Name:           cfg.GetName(),
+		EnterpriseMeta: *cfg.GetEnterpriseMeta(),
+	}
+
+	snap.APIGateway.FSCertificates.Set(ref, cfg)
+
+	return nil
+}
+
 // handleInlineCertConfigUpdate stores the certificate for the gateway
 func (h *handlerAPIGateway) handleInlineCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
 	resp, ok := u.Result.(*structs.ConfigEntryResponse)

agent/proxycfg/proxycfg.deepcopy.go

@@ -422,6 +422,7 @@ func (o *configSnapshotAPIGateway) DeepCopy() *configSnapshotAPIGateway {
 	cp.HTTPRoutes = o.HTTPRoutes.DeepCopy()
 	cp.TCPRoutes = o.TCPRoutes.DeepCopy()
 	cp.Certificates = o.Certificates.DeepCopy()
+	cp.FSCertificates = o.FSCertificates.DeepCopy()
 	if o.Listeners != nil {
 		cp.Listeners = make(map[string]structs.APIGatewayListener, len(o.Listeners))
 		for k2, v2 := range o.Listeners {

agent/proxycfg/snapshot.go

@@ -735,9 +735,12 @@ type configSnapshotAPIGateway struct {
 	// UpstreamsSet is the unique set of UpstreamID the gateway routes to.
 	UpstreamsSet routeUpstreamSet
 
-	HTTPRoutes   watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
-	TCPRoutes    watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]
-	Certificates watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
+	HTTPRoutes watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
+	TCPRoutes  watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]
+
+	// TODO(nathancoleman) Consider consolidating the 2 below into a map of structs.ConfigEntry
+	Certificates   watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
+	FSCertificates watch.Map[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]
 
 	// LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's
 	// leaf cert watch with different parameters.

agent/proxycfg/state.go

@@ -38,6 +38,7 @@ const (
 	gatewayConfigWatchID               = "gateway-config"
 	apiGatewayConfigWatchID            = "api-gateway-config"
 	boundGatewayConfigWatchID          = "bound-gateway-config"
+	fileSystemCertificateConfigWatchID = "file-system-certificate-config"
 	inlineCertificateConfigWatchID     = "inline-certificate-config"
 	routeConfigWatchID                 = "route-config"
 	externalServiceIDPrefix            = "external-service:"

agent/structs/config_entry.go

@@ -27,22 +27,23 @@ import (
 )
 
 const (
-	ServiceDefaults    string = "service-defaults"
-	ProxyDefaults      string = "proxy-defaults"
-	ServiceRouter      string = "service-router"
-	ServiceSplitter    string = "service-splitter"
-	ServiceResolver    string = "service-resolver"
-	IngressGateway     string = "ingress-gateway"
-	TerminatingGateway string = "terminating-gateway"
-	ServiceIntentions  string = "service-intentions"
-	MeshConfig         string = "mesh"
-	ExportedServices   string = "exported-services"
-	SamenessGroup      string = "sameness-group"
-	APIGateway         string = "api-gateway"
-	BoundAPIGateway    string = "bound-api-gateway"
-	InlineCertificate  string = "inline-certificate"
-	HTTPRoute          string = "http-route"
-	TCPRoute           string = "tcp-route"
+	ServiceDefaults       string = "service-defaults"
+	ProxyDefaults         string = "proxy-defaults"
+	ServiceRouter         string = "service-router"
+	ServiceSplitter       string = "service-splitter"
+	ServiceResolver       string = "service-resolver"
+	IngressGateway        string = "ingress-gateway"
+	TerminatingGateway    string = "terminating-gateway"
+	ServiceIntentions     string = "service-intentions"
+	MeshConfig            string = "mesh"
+	ExportedServices      string = "exported-services"
+	SamenessGroup         string = "sameness-group"
+	APIGateway            string = "api-gateway"
+	BoundAPIGateway       string = "bound-api-gateway"
+	FileSystemCertificate string = "file-system-certificate"
+	InlineCertificate     string = "inline-certificate"
+	HTTPRoute             string = "http-route"
+	TCPRoute              string = "tcp-route"
 	// TODO: decide if we want to highlight 'ip' keyword in the name of RateLimitIPConfig
 	RateLimitIPConfig string = "control-plane-request-limit"
 	JWTProvider       string = "jwt-provider"
@@ -71,6 +72,7 @@ var AllConfigEntryKinds = []string{
 	BoundAPIGateway,
 	HTTPRoute,
 	TCPRoute,
+	FileSystemCertificate,
 	InlineCertificate,
 	RateLimitIPConfig,
 	JWTProvider,
@@ -832,6 +834,8 @@ func MakeConfigEntry(kind, name string) (ConfigEntry, error) {
 		return &APIGatewayConfigEntry{Name: name}, nil
 	case BoundAPIGateway:
 		return &BoundAPIGatewayConfigEntry{Name: name}, nil
+	case FileSystemCertificate:
+		return &FileSystemCertificateConfigEntry{Name: name}, nil
 	case InlineCertificate:
 		return &InlineCertificateConfigEntry{Name: name}, nil
 	case HTTPRoute: