Skip to content

Commit

Permalink
Add SNI support so we can have multiple certificates
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrew Stucki committed Feb 17, 2023
1 parent 89a4cc5 commit 3da3bce
Show file tree
Hide file tree
Showing 14 changed files with 612 additions and 107 deletions.
24 changes: 24 additions & 0 deletions agent/structs/config_entry_inline_certificate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,30 @@ func (e *InlineCertificateConfigEntry) Validate() error {
return nil
}

func (e *InlineCertificateConfigEntry) Hosts() ([]string, error) {
certificateBlock, _ := pem.Decode([]byte(e.Certificate))
if certificateBlock == nil {
return nil, errors.New("failed to parse certificate PEM")
}

certificate, err := x509.ParseCertificate(certificateBlock.Bytes)
if err != nil {
return nil, fmt.Errorf("failed to parse certificate: %w", err)
}

hosts := []string{certificate.Subject.CommonName}

for _, name := range certificate.DNSNames {
hosts = append(hosts, name)
}

for _, ip := range certificate.IPAddresses {
hosts = append(hosts, ip.String())
}

return hosts, nil
}

func (e *InlineCertificateConfigEntry) CanRead(authz acl.Authorizer) error {
var authzContext acl.AuthorizerContext
e.FillAuthzContext(&authzContext)
Expand Down
3 changes: 3 additions & 0 deletions agent/xds/listeners.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2328,6 +2328,9 @@ func makeHTTPInspectorListenerFilter() (*envoy_listener_v3.ListenerFilter, error
}

func makeSNIFilterChainMatch(sniMatches ...string) *envoy_listener_v3.FilterChainMatch {
if sniMatches == nil {
return nil
}
return &envoy_listener_v3.FilterChainMatch{
ServerNames: sniMatches,
}
Expand Down
186 changes: 152 additions & 34 deletions agent/xds/listeners_ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,16 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
return nil, fmt.Errorf("no listener config found for listener on proto/port %s/%d", listenerKey.Protocol, listenerKey.Port)
}

var isAPIGatewayWithTLS bool
var certs []structs.InlineCertificateConfigEntry
if cfgSnap.APIGateway.ListenerCertificates != nil {
certs = cfgSnap.APIGateway.ListenerCertificates[listenerKey]
}
if certs != nil {
isAPIGatewayWithTLS = true
}

tlsContext, err := makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg, certs)
tlsContext, err := makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -78,6 +82,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
logger: s.Logger,
}
l := makeListener(opts)

filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{
accessLogs: &cfgSnap.Proxy.AccessLogs,
routeName: uid.EnvoyID(),
Expand All @@ -93,8 +98,23 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
l.FilterChains = []*envoy_listener_v3.FilterChain{
filterChain,
}
resources = append(resources, l)

if isAPIGatewayWithTLS {
l.FilterChains, err = makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey, listenerFilterOpts{
useRDS: useRDS,
protocol: listenerKey.Protocol,
routeName: listenerKey.RouteName(),
cluster: clusterName,
statPrefix: "ingress_upstream_",
accessLogs: &cfgSnap.Proxy.AccessLogs,
logger: s.Logger,
}, certs)
if err != nil {
return nil, err
}
}

resources = append(resources, l)
} else {
// If multiple upstreams share this port, make a special listener for the protocol.
listenerOpts := makeListenerOpts{
Expand Down Expand Up @@ -127,6 +147,13 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
return nil, err
}

if isAPIGatewayWithTLS {
sniFilterChains, err = makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey, filterOpts, certs)
if err != nil {
return nil, err
}
}

// If there are any sni filter chains, we need a TLS inspector filter!
if len(sniFilterChains) > 0 {
tlsInspector, err := makeTLSInspectorListenerFilter()
Expand All @@ -140,7 +167,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap

// See if there are other services that didn't have specific SNI-matching
// filter chains. If so add a default filterchain to serve them.
if len(sniFilterChains) < len(upstreams) {
if len(sniFilterChains) < len(upstreams) && !isAPIGatewayWithTLS {
defaultFilter, err := makeListenerFilter(filterOpts)
if err != nil {
return nil, err
Expand All @@ -166,10 +193,10 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
return resources, nil
}

func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener, certs []structs.InlineCertificateConfigEntry) (*envoy_tls_v3.DownstreamTlsContext, error) {
func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener) (*envoy_tls_v3.DownstreamTlsContext, error) {
var downstreamContext *envoy_tls_v3.DownstreamTlsContext

tlsContext, err := makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg, certs)
tlsContext, err := makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg)
if err != nil {
return nil, err
}
Expand All @@ -187,7 +214,7 @@ func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.Config
return downstreamContext, nil
}

func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener, certs []structs.InlineCertificateConfigEntry) (*envoy_tls_v3.CommonTlsContext, error) {
func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener) (*envoy_tls_v3.CommonTlsContext, error) {
var tlsContext *envoy_tls_v3.CommonTlsContext

// Enable connect TLS if it is enabled at the Gateway or specific listener
Expand All @@ -205,10 +232,6 @@ func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnap
return nil, err
}

if len(certs) != 0 {
return makeInlineTLSContextFromGatewayTLSConfig(*tlsCfg, certs), nil
}

if tlsCfg.SDS != nil {
// Set up listener TLS from SDS
tlsContext = makeCommonTLSContextFromGatewayTLSConfig(*tlsCfg)
Expand Down Expand Up @@ -389,15 +412,128 @@ func makeSDSOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
return chains, nil
}

// when we have multiple certificates on a single listener, we need
// to duplicate the filter chains with multiple TLS contexts
func makeInlineOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
tlsCfg structs.GatewayTLSConfig,
listenerKey proxycfg.IngressListenerKey,
filterOpts listenerFilterOpts,
certs []structs.InlineCertificateConfigEntry) ([]*envoy_listener_v3.FilterChain, error) {

listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey]
if !ok {
return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port)
}

var chains []*envoy_listener_v3.FilterChain

constructChain := func(name string, hosts []string, tlsContext *envoy_tls_v3.CommonTlsContext) error {
filterOpts.filterName = name
filter, err := makeListenerFilter(filterOpts)
if err != nil {
return err
}

transportSocket, err := makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: tlsContext,
RequireClientCertificate: &wrapperspb.BoolValue{Value: false},
})
if err != nil {
return err
}

chains = append(chains, &envoy_listener_v3.FilterChain{
FilterChainMatch: makeSNIFilterChainMatch(hosts...),
Filters: []*envoy_listener_v3.Filter{
filter,
},
TransportSocket: transportSocket,
})

return nil
}

multipleCerts := len(certs) > 1

allCertHosts := map[string]struct{}{}
overlappingHosts := map[string]struct{}{}

if multipleCerts {
// we only need to prune out overlapping hosts if we have more than
// one certificate
for _, cert := range certs {
hosts, err := cert.Hosts()
if err != nil {
return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
}
for _, host := range hosts {
if _, ok := allCertHosts[host]; ok {
overlappingHosts[host] = struct{}{}
}
allCertHosts[host] = struct{}{}
}
}
// make allCertHosts contain only certs that we
// should bind for a particular cert
for host := range overlappingHosts {
delete(allCertHosts, host)
}
}

for _, cert := range certs {
var hosts []string

// if we only have one cert, we just use it for all ingress
if multipleCerts {
// otherwise, we need an SNI per cert and to fallback to our ingress
// gateway certificate signed by our Consul CA
certHosts, err := cert.Hosts()
if err != nil {
return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
}
// filter out any overlapping hosts so we don't have collisions in our filter chains
for _, host := range certHosts {
if _, ok := overlappingHosts[host]; !ok {
hosts = append(hosts, host)
}
}

if len(hosts) == 0 {
// all of our hosts are overlapping, so we just skip this filter and it'll be
// handled by the default filter chain
continue
}
}

tlsContext := makeInlineTLSContextFromGatewayTLSConfig(tlsCfg, cert)
if multipleCerts {
// Configure alpn protocols on TLSContext
tlsContext.AlpnProtocols = getAlpnProtocols(listenerCfg.Protocol)
}

if err := constructChain(cert.Name, hosts, tlsContext); err != nil {
return nil, err
}
}

if multipleCerts {
// if we have more than one cert, add a default handler that uses the leaf cert from connect
if err := constructChain("default", nil, makeCommonTLSContext(cfgSnap.Leaf(), cfgSnap.RootPEMs(), makeTLSParametersFromGatewayTLSConfig(tlsCfg))); err != nil {
return nil, err
}
}

return chains, nil
}

func makeTLSParametersFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *envoy_tls_v3.TlsParameters {
return makeTLSParametersFromTLSConfig(tlsCfg.TLSMinVersion, tlsCfg.TLSMaxVersion, tlsCfg.CipherSuites)
}

func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, certs []structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
tlsParams := makeTLSParametersFromGatewayTLSConfig(tlsCfg)
tlsCerts := []*envoy_tls_v3.TlsCertificate{}
for _, cert := range certs {
tlsCerts = append(tlsCerts, &envoy_tls_v3.TlsCertificate{
func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, cert structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
return &envoy_tls_v3.CommonTlsContext{
TlsParams: makeTLSParametersFromGatewayTLSConfig(tlsCfg),
TlsCertificates: []*envoy_tls_v3.TlsCertificate{{
CertificateChain: &envoy_core_v3.DataSource{
Specifier: &envoy_core_v3.DataSource_InlineString{
InlineString: lib.EnsureTrailingNewline(cert.Certificate),
Expand All @@ -408,11 +544,7 @@ func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, c
InlineString: lib.EnsureTrailingNewline(cert.PrivateKey),
},
},
})
}
return &envoy_tls_v3.CommonTlsContext{
TlsParams: tlsParams,
TlsCertificates: tlsCerts,
}},
}
}

Expand All @@ -430,20 +562,6 @@ func makeCommonTLSContextFromGatewayServiceTLSConfig(tlsCfg structs.GatewayServi
}
}

func makeTLSCertificateSdsSecretConfigsFromSDSUsingADS(sdsCfg structs.GatewayTLSSDSConfig) []*envoy_tls_v3.SdsSecretConfig {
return []*envoy_tls_v3.SdsSecretConfig{
{
Name: sdsCfg.CertResource,
SdsConfig: &envoy_core_v3.ConfigSource{
ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{
Ads: &envoy_core_v3.AggregatedConfigSource{},
},
ResourceApiVersion: envoy_core_v3.ApiVersion_V3,
},
},
}
}

func makeTLSCertificateSdsSecretConfigsFromSDS(sdsCfg structs.GatewayTLSSDSConfig) []*envoy_tls_v3.SdsSecretConfig {
return []*envoy_tls_v3.SdsSecretConfig{
{
Expand Down
Loading

0 comments on commit 3da3bce

Please sign in to comment.