Skip to content

Commit

Permalink
[NET-8091] Add file-system-certificate config entry for API gateway (h…
Browse files Browse the repository at this point in the history
…ashicorp#20873)

* Define file-system-certificate config entry

* Collect file-system-certificate(s) referenced by api-gateway onto snapshot

* Add file-system-certificate to config entry kind allow lists

* Remove inapplicable validation

This validation makes sense for inline certificates since Consul server is holding the certificate; however, for file system certificates, Consul server never actually sees the certificate.

* Support file-system-certificate as source for listener TLS certificate

* Add more required mappings for the new config entry type

* Construct proper TLS context based on certificate kind

* Add support or SDS in xdscommon

* Remove unused param

* Adds back verification of certs for inline-certificates

* Undo tangential changes to TLS config consumption

* Remove stray curly braces

* Undo some more tangential changes

* Improve function name for generating API gateway secrets

* Add changelog entry

* Update .changelog/20873.txt

Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com>

* Add some nil-checking, remove outdated TODO

* Update test assertions to include file-system-certificate

* Add documentation for file-system-certificate config entry

Add new doc to nav

* Fix grammar mistake

* Rename watchmaps, remove outdated TODO

---------

Co-authored-by: Melisa Griffin <melisa.griffin@hashicorp.com>
Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com>
  • Loading branch information
3 people authored Apr 15, 2024
1 parent e52b170 commit 5e9f02d
Show file tree
Hide file tree
Showing 37 changed files with 2,838 additions and 2,169 deletions.
3 changes: 3 additions & 0 deletions .changelog/20873.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:feature
gateways: api-gateway can leverage listener TLS certificates available on the gateway's local filesystem by specifying the public certificate and private key path in the new file-system-certificate configuration entry
```
11 changes: 11 additions & 0 deletions agent/consul/fsm/decode_downgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,8 @@ func MakeShadowConfigEntry(kind, name string) (structs.ConfigEntry, error) {
return &ShadowAPIGatewayConfigEntry{APIGatewayConfigEntry: &structs.APIGatewayConfigEntry{Name: name}}, nil
case structs.BoundAPIGateway:
return &ShadowBoundAPIGatewayConfigEntry{BoundAPIGatewayConfigEntry: &structs.BoundAPIGatewayConfigEntry{Name: name}}, nil
case structs.FileSystemCertificate:
return &ShadowFileSystemCertificateConfigEntry{FileSystemCertificateConfigEntry: &structs.FileSystemCertificateConfigEntry{Name: name}}, nil
case structs.InlineCertificate:
return &ShadowInlineCertificateConfigEntry{InlineCertificateConfigEntry: &structs.InlineCertificateConfigEntry{Name: name}}, nil
case structs.HTTPRoute:
Expand Down Expand Up @@ -931,6 +933,15 @@ func (s ShadowBoundAPIGatewayConfigEntry) GetRealConfigEntry() structs.ConfigEnt
return s.BoundAPIGatewayConfigEntry
}

type ShadowFileSystemCertificateConfigEntry struct {
ShadowBase
*structs.FileSystemCertificateConfigEntry
}

func (s ShadowFileSystemCertificateConfigEntry) GetRealConfigEntry() structs.ConfigEntry {
return s.FileSystemCertificateConfigEntry
}

type ShadowInlineCertificateConfigEntry struct {
ShadowBase
*structs.InlineCertificateConfigEntry
Expand Down
5 changes: 5 additions & 0 deletions agent/consul/fsm/fsm.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,11 @@ func (c *FSM) registerStreamSnapshotHandlers() {
}, true)
panicIfErr(err)

err = c.deps.Publisher.RegisterHandler(state.EventTopicFileSystemCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
return c.State().FileSystemCertificateSnapshot(req, buf)
}, true)
panicIfErr(err)

err = c.deps.Publisher.RegisterHandler(state.EventTopicInlineCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
return c.State().InlineCertificateSnapshot(req, buf)
}, true)
Expand Down
2 changes: 1 addition & 1 deletion agent/consul/gateways/controller_gateways.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Req
return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileHTTPRoute, r.cleanupRoute)
case structs.TCPRoute:
return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileTCPRoute, r.cleanupRoute)
case structs.InlineCertificate:
case structs.InlineCertificate, structs.FileSystemCertificate:
return r.enqueueCertificateReferencedGateways(r.fsm.State(), ctx, req)
case structs.JWTProvider:
return r.enqueueJWTProviderReferencedGatewaysAndHTTPRoutes(r.fsm.State(), ctx, req)
Expand Down
1 change: 1 addition & 0 deletions agent/consul/state/config_entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -646,6 +646,7 @@ func validateProposedConfigEntryInGraph(
case structs.ExportedServices:
case structs.APIGateway: // TODO Consider checkGatewayClash
case structs.BoundAPIGateway:
case structs.FileSystemCertificate:
case structs.InlineCertificate:
case structs.HTTPRoute:
case structs.TCPRoute:
Expand Down
35 changes: 21 additions & 14 deletions agent/consul/state/config_entry_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@ import (

// Adding events for a new config entry kind? Remember to update ConfigEntryFromStructs and ConfigEntryToStructs.
var configEntryKindToTopic = map[string]stream.Topic{
structs.MeshConfig: EventTopicMeshConfig,
structs.ServiceResolver: EventTopicServiceResolver,
structs.IngressGateway: EventTopicIngressGateway,
structs.ServiceIntentions: EventTopicServiceIntentions,
structs.ServiceDefaults: EventTopicServiceDefaults,
structs.APIGateway: EventTopicAPIGateway,
structs.TCPRoute: EventTopicTCPRoute,
structs.HTTPRoute: EventTopicHTTPRoute,
structs.InlineCertificate: EventTopicInlineCertificate,
structs.BoundAPIGateway: EventTopicBoundAPIGateway,
structs.RateLimitIPConfig: EventTopicIPRateLimit,
structs.SamenessGroup: EventTopicSamenessGroup,
structs.JWTProvider: EventTopicJWTProvider,
structs.ExportedServices: EventTopicExportedServices,
structs.MeshConfig: EventTopicMeshConfig,
structs.ServiceResolver: EventTopicServiceResolver,
structs.IngressGateway: EventTopicIngressGateway,
structs.ServiceIntentions: EventTopicServiceIntentions,
structs.ServiceDefaults: EventTopicServiceDefaults,
structs.APIGateway: EventTopicAPIGateway,
structs.TCPRoute: EventTopicTCPRoute,
structs.HTTPRoute: EventTopicHTTPRoute,
structs.FileSystemCertificate: EventTopicFileSystemCertificate,
structs.InlineCertificate: EventTopicInlineCertificate,
structs.BoundAPIGateway: EventTopicBoundAPIGateway,
structs.RateLimitIPConfig: EventTopicIPRateLimit,
structs.SamenessGroup: EventTopicSamenessGroup,
structs.JWTProvider: EventTopicJWTProvider,
structs.ExportedServices: EventTopicExportedServices,
}

// EventSubjectConfigEntry is a stream.Subject used to route and receive events
Expand Down Expand Up @@ -147,6 +148,12 @@ func (s *Store) HTTPRouteSnapshot(req stream.SubscribeRequest, buf stream.Snapsh
return s.configEntrySnapshot(structs.HTTPRoute, req, buf)
}

// FileSystemCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
// inline-certificate config entries.
func (s *Store) FileSystemCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
return s.configEntrySnapshot(structs.FileSystemCertificate, req, buf)
}

// InlineCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
// inline-certificate config entries.
func (s *Store) InlineCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
Expand Down
3 changes: 2 additions & 1 deletion agent/consul/state/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ func PBToStreamSubscribeRequest(req *pbsubscribe.SubscribeRequest, entMeta acl.E
case EventTopicMeshConfig, EventTopicServiceResolver, EventTopicIngressGateway,
EventTopicServiceIntentions, EventTopicServiceDefaults, EventTopicAPIGateway,
EventTopicTCPRoute, EventTopicHTTPRoute, EventTopicJWTProvider, EventTopicInlineCertificate,
EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices:
EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices,
EventTopicFileSystemCertificate:
subject = EventSubjectConfigEntry{
Name: named.Key,
EnterpriseMeta: &entMeta,
Expand Down
35 changes: 18 additions & 17 deletions agent/consul/state/memdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,23 +196,24 @@ func (db *readDB) ReadTxn() AbortTxn {
}

var (
EventTopicServiceHealth = pbsubscribe.Topic_ServiceHealth
EventTopicServiceHealthConnect = pbsubscribe.Topic_ServiceHealthConnect
EventTopicMeshConfig = pbsubscribe.Topic_MeshConfig
EventTopicServiceResolver = pbsubscribe.Topic_ServiceResolver
EventTopicIngressGateway = pbsubscribe.Topic_IngressGateway
EventTopicServiceIntentions = pbsubscribe.Topic_ServiceIntentions
EventTopicServiceDefaults = pbsubscribe.Topic_ServiceDefaults
EventTopicServiceList = pbsubscribe.Topic_ServiceList
EventTopicAPIGateway = pbsubscribe.Topic_APIGateway
EventTopicTCPRoute = pbsubscribe.Topic_TCPRoute
EventTopicHTTPRoute = pbsubscribe.Topic_HTTPRoute
EventTopicInlineCertificate = pbsubscribe.Topic_InlineCertificate
EventTopicBoundAPIGateway = pbsubscribe.Topic_BoundAPIGateway
EventTopicIPRateLimit = pbsubscribe.Topic_IPRateLimit
EventTopicSamenessGroup = pbsubscribe.Topic_SamenessGroup
EventTopicJWTProvider = pbsubscribe.Topic_JWTProvider
EventTopicExportedServices = pbsubscribe.Topic_ExportedServices
EventTopicServiceHealth = pbsubscribe.Topic_ServiceHealth
EventTopicServiceHealthConnect = pbsubscribe.Topic_ServiceHealthConnect
EventTopicMeshConfig = pbsubscribe.Topic_MeshConfig
EventTopicServiceResolver = pbsubscribe.Topic_ServiceResolver
EventTopicIngressGateway = pbsubscribe.Topic_IngressGateway
EventTopicServiceIntentions = pbsubscribe.Topic_ServiceIntentions
EventTopicServiceDefaults = pbsubscribe.Topic_ServiceDefaults
EventTopicServiceList = pbsubscribe.Topic_ServiceList
EventTopicAPIGateway = pbsubscribe.Topic_APIGateway
EventTopicTCPRoute = pbsubscribe.Topic_TCPRoute
EventTopicHTTPRoute = pbsubscribe.Topic_HTTPRoute
EventTopicFileSystemCertificate = pbsubscribe.Topic_FileSystemCertificate
EventTopicInlineCertificate = pbsubscribe.Topic_InlineCertificate
EventTopicBoundAPIGateway = pbsubscribe.Topic_BoundAPIGateway
EventTopicIPRateLimit = pbsubscribe.Topic_IPRateLimit
EventTopicSamenessGroup = pbsubscribe.Topic_SamenessGroup
EventTopicJWTProvider = pbsubscribe.Topic_JWTProvider
EventTopicExportedServices = pbsubscribe.Topic_ExportedServices
)

func processDBChanges(tx ReadTxn, changes Changes) ([]stream.Event, error) {
Expand Down
16 changes: 16 additions & 0 deletions agent/consul/usagemetrics/usagemetrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,14 @@ var baseCases = map[string]testCase{
{Name: "kind", Value: "bound-api-gateway"},
},
},
"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
Name: "consul.usage.test.state.config_entries",
Value: 0,
Labels: []metrics.Label{
{Name: "datacenter", Value: "dc1"},
{Name: "kind", Value: "file-system-certificate"},
},
},
"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
Name: "consul.usage.test.state.config_entries",
Value: 0,
Expand Down Expand Up @@ -607,6 +615,14 @@ var baseCases = map[string]testCase{
{Name: "kind", Value: "bound-api-gateway"},
},
},
"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
Name: "consul.usage.test.state.config_entries",
Value: 0,
Labels: []metrics.Label{
{Name: "datacenter", Value: "dc1"},
{Name: "kind", Value: "file-system-certificate"},
},
},
"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
Name: "consul.usage.test.state.config_entries",
Value: 0,
Expand Down
2 changes: 2 additions & 0 deletions agent/proxycfg-glue/config_entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ func newConfigEntryRequest(req *structs.ConfigEntryQuery, deps ServerDataSourceD
topic = pbsubscribe.Topic_HTTPRoute
case structs.TCPRoute:
topic = pbsubscribe.Topic_TCPRoute
case structs.FileSystemCertificate:
topic = pbsubscribe.Topic_FileSystemCertificate
case structs.InlineCertificate:
topic = pbsubscribe.Topic_InlineCertificate
case structs.BoundAPIGateway:
Expand Down
67 changes: 57 additions & 10 deletions agent/proxycfg/api_gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err
snap.APIGateway.BoundListeners = make(map[string]structs.BoundAPIGatewayListener)
snap.APIGateway.HTTPRoutes = watch.NewMap[structs.ResourceReference, *structs.HTTPRouteConfigEntry]()
snap.APIGateway.TCPRoutes = watch.NewMap[structs.ResourceReference, *structs.TCPRouteConfigEntry]()
snap.APIGateway.Certificates = watch.NewMap[structs.ResourceReference, *structs.InlineCertificateConfigEntry]()
snap.APIGateway.InlineCertificates = watch.NewMap[structs.ResourceReference, *structs.InlineCertificateConfigEntry]()
snap.APIGateway.FileSystemCertificates = watch.NewMap[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]()

snap.APIGateway.Upstreams = make(listenerRouteUpstreams)
snap.APIGateway.UpstreamsSet = make(routeUpstreamSet)
Expand Down Expand Up @@ -96,7 +97,8 @@ func (h *handlerAPIGateway) subscribeToConfigEntry(ctx context.Context, kind, na
// handleUpdate responds to changes in the api-gateway. In general, we want
// to crawl the various resources related to or attached to the gateway and
// collect the list of things need to generate xDS. This list of resources
// includes the bound-api-gateway, http-routes, tcp-routes, and inline-certificates.
// includes the bound-api-gateway, http-routes, tcp-routes,
// file-system-certificates and inline-certificates.
func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
if u.Err != nil {
return fmt.Errorf("error filling agent cache: %v", u.Err)
Expand All @@ -113,6 +115,11 @@ func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, sna
if err := h.handleGatewayConfigUpdate(ctx, u, snap, u.CorrelationID); err != nil {
return err
}
case fileSystemCertificateConfigWatchID:
// Handle change in an attached file-system-certificate config entry
if err := h.handleFileSystemCertConfigUpdate(ctx, u, snap); err != nil {
return err
}
case inlineCertificateConfigWatchID:
// Handle change in an attached inline-certificate config entry
if err := h.handleInlineCertConfigUpdate(ctx, u, snap); err != nil {
Expand Down Expand Up @@ -205,12 +212,21 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
for _, ref := range listener.Certificates {
ctx, cancel := context.WithCancel(ctx)
seenRefs[ref] = struct{}{}
snap.APIGateway.Certificates.InitWatch(ref, cancel)

err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
if err != nil {
// TODO May want to continue
return err
if ref.Kind == structs.FileSystemCertificate {
snap.APIGateway.FileSystemCertificates.InitWatch(ref, cancel)

err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, fileSystemCertificateConfigWatchID)
if err != nil {
return err
}
} else {
snap.APIGateway.InlineCertificates.InitWatch(ref, cancel)

err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
if err != nil {
return err
}
}
}
}
Expand All @@ -234,9 +250,16 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
return true
})

snap.APIGateway.Certificates.ForEachKey(func(ref structs.ResourceReference) bool {
snap.APIGateway.InlineCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
if _, ok := seenRefs[ref]; !ok {
snap.APIGateway.Certificates.CancelWatch(ref)
snap.APIGateway.InlineCertificates.CancelWatch(ref)
}
return true
})

snap.APIGateway.FileSystemCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
if _, ok := seenRefs[ref]; !ok {
snap.APIGateway.FileSystemCertificates.CancelWatch(ref)
}
return true
})
Expand Down Expand Up @@ -265,6 +288,30 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
return h.watchIngressLeafCert(ctx, snap)
}

func (h *handlerAPIGateway) handleFileSystemCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
resp, ok := u.Result.(*structs.ConfigEntryResponse)
if !ok {
return fmt.Errorf("invalid type for response: %T", u.Result)
} else if resp == nil || resp.Entry == nil {
return nil
}

cfg, ok := resp.Entry.(*structs.FileSystemCertificateConfigEntry)
if !ok || cfg == nil {
return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
}

ref := structs.ResourceReference{
Kind: cfg.GetKind(),
Name: cfg.GetName(),
EnterpriseMeta: *cfg.GetEnterpriseMeta(),
}

snap.APIGateway.FileSystemCertificates.Set(ref, cfg)

return nil
}

// handleInlineCertConfigUpdate stores the certificate for the gateway
func (h *handlerAPIGateway) handleInlineCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
resp, ok := u.Result.(*structs.ConfigEntryResponse)
Expand All @@ -285,7 +332,7 @@ func (h *handlerAPIGateway) handleInlineCertConfigUpdate(_ context.Context, u Up
EnterpriseMeta: *cfg.GetEnterpriseMeta(),
}

snap.APIGateway.Certificates.Set(ref, cfg)
snap.APIGateway.InlineCertificates.Set(ref, cfg)

return nil
}
Expand Down
3 changes: 2 additions & 1 deletion agent/proxycfg/proxycfg.deepcopy.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,8 @@ func (o *configSnapshotAPIGateway) DeepCopy() *configSnapshotAPIGateway {
}
cp.HTTPRoutes = o.HTTPRoutes.DeepCopy()
cp.TCPRoutes = o.TCPRoutes.DeepCopy()
cp.Certificates = o.Certificates.DeepCopy()
cp.InlineCertificates = o.InlineCertificates.DeepCopy()
cp.FileSystemCertificates = o.FileSystemCertificates.DeepCopy()
if o.Listeners != nil {
cp.Listeners = make(map[string]structs.APIGatewayListener, len(o.Listeners))
for k2, v2 := range o.Listeners {
Expand Down
11 changes: 7 additions & 4 deletions agent/proxycfg/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"sort"
"strings"

"github.com/hashicorp/go-hclog"

"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
Expand All @@ -17,7 +19,6 @@ import (
"github.com/hashicorp/consul/agent/xds/config"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/proto/private/pbpeering"
"github.com/hashicorp/go-hclog"
)

// TODO(ingress): Can we think of a better for this bag of data?
Expand Down Expand Up @@ -735,9 +736,11 @@ type configSnapshotAPIGateway struct {
// UpstreamsSet is the unique set of UpstreamID the gateway routes to.
UpstreamsSet routeUpstreamSet

HTTPRoutes watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
TCPRoutes watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]
Certificates watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
HTTPRoutes watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
TCPRoutes watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]

InlineCertificates watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
FileSystemCertificates watch.Map[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]

// LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's
// leaf cert watch with different parameters.
Expand Down
1 change: 1 addition & 0 deletions agent/proxycfg/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ const (
gatewayConfigWatchID = "gateway-config"
apiGatewayConfigWatchID = "api-gateway-config"
boundGatewayConfigWatchID = "bound-gateway-config"
fileSystemCertificateConfigWatchID = "file-system-certificate-config"
inlineCertificateConfigWatchID = "inline-certificate-config"
routeConfigWatchID = "route-config"
externalServiceIDPrefix = "external-service:"
Expand Down
Loading

0 comments on commit 5e9f02d

Please sign in to comment.