diff --git a/Makefile b/Makefile index d28b92a96..5345ec34e 100644 --- a/Makefile +++ b/Makefile @@ -80,20 +80,20 @@ tail-envoyfleet: ## Tail logs of envoy .PHONY: enable-logging enable-logging: ## Set some particular logger's level kubectl port-forward --namespace default deployments/default 19000:19000 & echo $$! > /tmp/kube-port-forward-logging.pid - sleep 4 - curl -s -X POST "http://localhost:19000/logging?backtrace=trace" - curl -s -X POST "http://localhost:19000/logging?envoy_bug=trace" - curl -s -X POST "http://localhost:19000/logging?assert=trace" - curl -s -X POST "http://localhost:19000/logging?secret=trace" - curl -s -X POST "http://localhost:19000/logging?grpc=trace" - curl -s -X POST "http://localhost:19000/logging?ext_authz=trace" - curl -s -X POST "http://localhost:19000/logging?filter=trace" - curl -s -X POST "http://localhost:19000/logging?misc=trace" - curl -s -X POST "http://localhost:19000/logging?conn_handler=trace" - @# curl -s -X POST "http://localhost:19000/logging?connection=trace" - @# curl -s -X POST "http://localhost:19000/logging?http=trace" - @# curl -s -X POST "http://localhost:19000/logging?http2=trace" - @# curl -s -X POST "http://localhost:19000/logging?admin=trace" + sleep 2 + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?backtrace=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?envoy_bug=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?assert=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?secret=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?grpc=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?ext_authz=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?filter=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?misc=trace" + curl --silent --output /dev/null -X POST "http://localhost:19000/logging?conn_handler=trace" + @# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?connection=trace" + @# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http=trace" + @# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http2=trace" + @# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?admin=trace" @# bash -c "trap 'pkill -F /tmp/kube-port-forward-logging.pid' SIGINT SIGTERM ERR EXIT" @echo @echo "How to stop port forward to the admin port (19000):" diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 572be74ad..3c3749b00 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -267,15 +267,16 @@ func main() { }() secretsChan := make(chan *corev1.Secret) - controllerConfigManager := controllers.KubeEnvoyConfigManager{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EnvoyManager: envoyManager, - Validator: proxy, - SecretToEnvoyFleet: map[string]gateway.EnvoyFleetID{}, - WatchedSecretsChan: secretsChan, - OpenApiParser: spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}), - } + + controllerConfigManager := controllers.NewKubeEnvoyConfigManager( + mgr.GetClient(), + mgr.GetScheme(), + envoyManager, + proxy, + secretsChan, + map[string]gateway.EnvoyFleetID{}, + spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}), + ) analytics.SendAnonymousInfo(ctx, controllerConfigManager.Client, "kusk", "kusk-gateway manager bootstrapping") heartBeat(ctx, controllerConfigManager.Client) @@ -294,7 +295,7 @@ func main() { if err = (&controllers.EnvoyFleetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - ConfigManager: &controllerConfigManager, + ConfigManager: controllerConfigManager, }).SetupWithManager(mgr); err != nil { setupLog. WithValues("controller", "EnvoyFleet"). @@ -316,7 +317,7 @@ func main() { if err = (&controllers.APIReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - ConfigManager: &controllerConfigManager, + ConfigManager: controllerConfigManager, }).SetupWithManager(mgr); err != nil { setupLog. WithValues("controller", "API"). @@ -332,7 +333,7 @@ func main() { if err = (&controllers.StaticRouteReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - ConfigManager: &controllerConfigManager, + ConfigManager: controllerConfigManager, }).SetupWithManager(mgr); err != nil { setupLog. WithValues("controller", "StaticRoute"). diff --git a/config/samples/gateway_v1_envoyfleet.yaml b/config/samples/gateway_v1_envoyfleet.yaml index be43c32d4..fc990301d 100644 --- a/config/samples/gateway_v1_envoyfleet.yaml +++ b/config/samples/gateway_v1_envoyfleet.yaml @@ -78,7 +78,7 @@ spec: # The output example: # "[2021-12-15T16:50:50.217Z]" "GET" "/" "200" "1" text_template: | - "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%" + "[%START_TIME%]" "%REQ(:scheme)%" "%REQ(:METHOD)%" "%REQ(X-FORWARDED-PROTO)%://%REQ(:AUTHORITY)%%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" %BYTES_SENT% %DURATION% %UPSTREAM_HOST% %REQ(X-ENVOY-IP-TAGS)%" # Json format fields order isn't preserved # The output example: # {"start_time":"2021-12-15T16:46:52.135Z","path":"/","response_code":200,"method":"GET","duration":1} diff --git a/docs/docs/extension.md b/docs/docs/extension.md index 32caecdbe..fb1d9843a 100644 --- a/docs/docs/extension.md +++ b/docs/docs/extension.md @@ -327,7 +327,7 @@ The `auth` object contains the following properties to configure HTTP authentica | `auth.oauth2.authorization_endpoint` | **Required, if `scheme` is `oauth2`**. Defines the `authorization_endpoint`, e.g., the field `authorization_endpoint` from . | | `auth.oauth2.credentials.client_id` | **Required, if `scheme` is `oauth2`**. Defines the Client ID. | | `auth.oauth2.credentials.client_secret` | **Required, if `scheme` is `oauth2`**. Defines the Client Secret. | -| `auth.oauth2.redirect_uri` | **Required, if `scheme` is `oauth2`**. The redirect URI passed to the authorization endpoint. | +| `auth.oauth2.redirect_uri` | **Required, if `scheme` is `oauth2`**. The redirect URI passed to the authorization endpoint. It is advised to use `%REQ(x-forwarded-proto)%://%REQ(:authority)%` before the actual callback path, e.g., the `redirect_uri` should be defined as `""%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback/oauth2/callback"`, including the quotes. | | `auth.oauth2.signout_path` | **Required, if `scheme` is `oauth2`**. The path to sign a user out, clearing their credential cookies. | | `auth.oauth2.redirect_path_matcher` | **Required, if `scheme` is `oauth2`**. After a redirecting the user back to the `redirect_uri`, using this new grant and the `token_secret`, the `kusk-gateway` then attempts to retrieve an access token from the `token_endpoint`. The `kusk-gateway` knows it has to do this instead of reinitiating another login because the incoming request has a path that matches the `redirect_path_matcher` criteria. | | `auth.oauth2.forward_bearer_token` | **Required, if `scheme` is `oauth2`**. If the Bearer Token should be forwarded, you generally want this to be `true`. When the authn server validates the client and returns an authorization token back to `kusk-gateway`, no matter what format that token is, if `forward_bearer_token` is set to true `kusk-gateway` will send over a cookie named `BearerToken` to the upstream. Additionally, the `Authorization` header will be populated with the same value, i.e., Forward the OAuth token as a Bearer to upstream web service. | @@ -391,7 +391,7 @@ x-kusk: credentials: client_id: *CLIENT_ID* client_secret: *CLIENT_SECRET* - redirect_uri: /oauth2/callback + redirect_uri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback" redirect_path_matcher: /oauth2/callback signout_path: /oauth2/signout forward_bearer_token: true diff --git a/examples/auth/oauth2/authorization-code-grant/api.yaml b/examples/auth/oauth2/authorization-code-grant/api.yaml index 40a142863..8e60e9198 100644 --- a/examples/auth/oauth2/authorization-code-grant/api.yaml +++ b/examples/auth/oauth2/authorization-code-grant/api.yaml @@ -30,20 +30,21 @@ spec: credentials: client_id: upRN78W8GzV4TwFRp0ekZfLx2UnqJJs8 client_secret: Z6MX7NreJumWLmf6unsQ5uiEUrTBxfNtqG9Vy5Kjktnvfj-_fRCBO9EU1mL1YzAJ - redirect_uri: /oauth2/callback + #redirect_uri: "%REQ(:scheme)%://%REQ(:authority)%/oauth2/callback" + redirect_uri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback" redirect_path_matcher: /oauth2/callback signout_path: /oauth2/signout forward_bearer_token: true auth_scopes: - openid paths: - "/": - get: - description: Returns GET data. - operationId: "/get" - responses: {} "/uuid": get: description: Returns UUID4. operationId: "/uuid" responses: {} + # "/": + # get: + # description: Returns GET data. + # operationId: "/get" + # responses: {} diff --git a/go.mod b/go.mod index 50e16c3f7..f8ef28d4c 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/google/go-cmp v0.5.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.0 github.com/gookit/color v1.5.2 github.com/gorilla/mux v1.8.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect diff --git a/internal/controllers/config_manager.go b/internal/controllers/config_manager.go index 763e68c38..ae1cfaee8 100644 --- a/internal/controllers/config_manager.go +++ b/internal/controllers/config_manager.go @@ -54,15 +54,26 @@ const ( // KubeEnvoyConfigManager manages all Envoy configurations parsing from CRDs type KubeEnvoyConfigManager struct { client.Client - Scheme *runtime.Scheme - EnvoyManager *manager.EnvoyConfigManager - Validator validation.ValidationUpdater - m sync.Mutex - + Scheme *runtime.Scheme + EnvoyManager *manager.EnvoyConfigManager + Validator validation.ValidationUpdater WatchedSecretsChan chan *v1.Secret SecretToEnvoyFleet map[string]gateway.EnvoyFleetID + OpenApiParser spec.Parser + m *sync.Mutex +} - OpenApiParser spec.Parser +func NewKubeEnvoyConfigManager(client client.Client, scheme *runtime.Scheme, envoyManager *manager.EnvoyConfigManager, validator validation.ValidationUpdater, watchedSecretsChan chan *v1.Secret, secretToEnvoyFleet map[string]gateway.EnvoyFleetID, OpenApiParser spec.Parser) *KubeEnvoyConfigManager { + return &KubeEnvoyConfigManager{ + Client: client, + Scheme: scheme, + EnvoyManager: envoyManager, + Validator: validator, + WatchedSecretsChan: watchedSecretsChan, + SecretToEnvoyFleet: secretToEnvoyFleet, + OpenApiParser: OpenApiParser, + m: &sync.Mutex{}, + } } var ( @@ -71,7 +82,6 @@ var ( // UpdateConfiguration is the main method to gather all routing configs and to create and apply Envoy config func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetID gateway.EnvoyFleetID) error { - l := configManagerLogger fleetIDstr := fleetID.String() // acquiring this lock is required so that no potentially conflicting updates would happen at the same time @@ -82,7 +92,8 @@ func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetI l.Info("Started updating configuration", "fleet", fleetIDstr) defer l.Info("Finished updating configuration", "fleet", fleetIDstr) - envoyConfig := config.New() + envoyConfig := config.NewEnvoyConfiguration(ctrl.Log) + // fetch all APIs and Static Routes to rebuild Envoy configuration l.Info("Getting APIs for the fleet", "fleet", fleetIDstr) diff --git a/internal/controllers/envoy.yaml b/internal/controllers/envoy.yaml index 06c496c5f..560a6b472 100644 --- a/internal/controllers/envoy.yaml +++ b/internal/controllers/envoy.yaml @@ -16,23 +16,14 @@ dynamic_resources: ads: {} static_resources: - secrets: - - name: token - generic_secret: - secret: - inline_string: "" - - name: hmac - generic_secret: - secret: - inline_bytes: "KVPwEwpMmxmqxvT+tZu07vCDwpT41/vnjeM5kLW78Vc=" clusters: - - type: STRICT_DNS + - name: xds_cluster + type: STRICT_DNS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions explicit_http_config: http2_protocol_options: {} - name: xds_cluster load_assignment: cluster_name: xds_cluster endpoints: diff --git a/internal/envoy/auth/oauth2_filter.go b/internal/envoy/auth/oauth2_filter.go index 802bbd840..7af80a255 100644 --- a/internal/envoy/auth/oauth2_filter.go +++ b/internal/envoy/auth/oauth2_filter.go @@ -28,13 +28,16 @@ import ( "fmt" "net/url" "strconv" + "time" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_extensions_filter_http_oauth2_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/oauth2/v3" envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + "github.com/kubeshop/kusk-gateway/internal/envoy/types" "github.com/kubeshop/kusk-gateway/pkg/options" ) @@ -81,15 +84,35 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr } authorizationEndpoint := oauth2Options.AuthorizationEndpoint + sdsConfig := makeConfigSource() + + tokenSecretName, err := args.EnvoyConfiguration.AddGenericSecretString(oauth2Options.Credentials.ClientSecret) + if err != nil { + return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add %q, %w", types.TokenSecretType, err) + } + tokenSecret := &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{ - Name: "token", + Name: tokenSecretName, + SdsConfig: sdsConfig, + } + + hmac, err := GenerateHMAC() + if err != nil { + return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: cannot generate HMAC, %w", err) + } + + hmacSecretName, err := args.EnvoyConfiguration.AddGenericSecretBytes([]byte(hmac)) + if err != nil { + return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add %q, %w", types.HMACSecretType, err) } tokenFormation := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials_HmacSecret{ HmacSecret: &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{ - Name: "hmac", + Name: hmacSecretName, + SdsConfig: sdsConfig, }, } + credentials := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials{ // The client_id to be used in the authorize calls. This value will be URL encoded when sent to the OAuth server. ClientId: oauth2Options.Credentials.ClientID, @@ -102,8 +125,10 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr TokenFormation: tokenFormation, } + // // For example, `redirectUri` becomes "http://192.168.49.2/oauth2/callback" + // redirectUri := fmt.Sprintf("%s://%s%s", "%REQ(x-forwarded-proto)%", "%REQ(:authority)%", oauth2Options.RedirectURI) // For example, `redirectUri` becomes "http://192.168.49.2/oauth2/callback" - redirectUri := fmt.Sprintf("%s://%s%s", "%REQ(x-forwarded-proto)%", "%REQ(:authority)%", oauth2Options.RedirectURI) + redirectUri := oauth2Options.RedirectURI redirectPathMatcher := PathMatcherExact(oauth2Options.RedirectPathMatcher, false) signoutPath := PathMatcherExact(oauth2Options.SignoutPath, false) forwardBearerToken := oauth2Options.ForwardBearerToken @@ -156,6 +181,16 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr return anyOAuth2, nil } +func makeConfigSource() *envoy_config_core_v3.ConfigSource { + return &envoy_config_core_v3.ConfigSource{ + ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{ + Ads: &envoy_config_core_v3.AggregatedConfigSource{}, + }, + InitialFetchTimeout: durationpb.New(time.Second * 128), + ResourceApiVersion: ResourceApiVersion, + } +} + func GenerateHMAC() (string, error) { // Since HMAC use symmetric key algorithm, we can just generate random bytes as secret key. diff --git a/internal/envoy/config/config.go b/internal/envoy/config/envoy_configuration.go similarity index 54% rename from internal/envoy/config/config.go rename to internal/envoy/config/envoy_configuration.go index 77e4a8d8a..309bd841b 100644 --- a/internal/envoy/config/config.go +++ b/internal/envoy/config/envoy_configuration.go @@ -1,42 +1,44 @@ -/* -MIT License - -Copyright (c) 2022 Kubeshop - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ +// MIT License +// +// Copyright (c) 2022 Kubeshop +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + package config import ( "fmt" "sort" + "github.com/davecgh/go-spew/spew" cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" - cacheTypes "github.com/envoyproxy/go-control-plane/pkg/cache/types" - + envoy_cache_types "github.com/envoyproxy/go-control-plane/pkg/cache/types" "github.com/envoyproxy/go-control-plane/pkg/cache/v3" - "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + google_uuid "github.com/google/uuid" + + "github.com/go-logr/logr" "github.com/gofrs/uuid" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" @@ -44,6 +46,14 @@ import ( "github.com/kubeshop/kusk-gateway/internal/envoy/types" ) +// type SecretType string + +// const ( +// TokenSecretType SecretType = "token_secret" +// HMACSecretType SecretType = "hmac_secret" +// ) + +// EnvoyConfiguration // Simplified objects hierarchy configuration as for the static Envoy config // Top level objects are "listeners" and "clusters" // @@ -74,18 +84,21 @@ import ( // address: backendsvc1-dns-name // port_value: backendsvc1-port // - type EnvoyConfiguration struct { // vhosts maps vhost domain name or domain pattern to the list of vhosts assigned to the listener vHosts map[string]*types.VirtualHost clusters map[string]*cluster.Cluster listener *listener.Listener + secrets map[string]*envoy_extensions_transport_sockets_tls_v3.Secret + logger logr.Logger } -func New() *EnvoyConfiguration { +func NewEnvoyConfiguration(logger logr.Logger) *EnvoyConfiguration { return &EnvoyConfiguration{ clusters: make(map[string]*cluster.Cluster), vHosts: make(map[string]*types.VirtualHost), + secrets: map[string]*envoy_extensions_transport_sockets_tls_v3.Secret{}, + logger: logger.WithName("EnvoyConfiguration"), } } @@ -156,9 +169,9 @@ func (e *EnvoyConfiguration) AddClusterWithTLS(clusterName, upstreamServiceHost return fmt.Errorf("EnvoyConfiguration.AddClusterWithTLS: failed on `anypb.New(upstreamTlsContext)`, %w", err) } - transportSocket := &core.TransportSocket{ + transportSocket := &envoy_config_core_v3.TransportSocket{ Name: "envoy.transport_sockets.tls", - ConfigType: &core.TransportSocket_TypedConfig{ + ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{ TypedConfig: anyUpstreamTlsContext, }, } @@ -171,7 +184,7 @@ func (e *EnvoyConfiguration) AddClusterWithTLS(clusterName, upstreamServiceHost LoadAssignment: loadAssignment, DnsLookupFamily: cluster.Cluster_V4_ONLY, TransportSocket: transportSocket, - UpstreamHttpProtocolOptions: &core.UpstreamHttpProtocolOptions{ + UpstreamHttpProtocolOptions: &envoy_config_core_v3.UpstreamHttpProtocolOptions{ // Set transport socket `SNI `_ for new // upstream connections based on the downstream HTTP host/authority header or any other arbitrary // header when :ref:`override_auto_sni_header ` @@ -197,12 +210,12 @@ func createLoadAssignment(clusterName string, upstreamServiceHost string, upstre { HostIdentifier: &endpoint.LbEndpoint_Endpoint{ Endpoint: &endpoint.Endpoint{ - Address: &core.Address{ - Address: &core.Address_SocketAddress{ - SocketAddress: &core.SocketAddress{ - Protocol: core.SocketAddress_TCP, + Address: &envoy_config_core_v3.Address{ + Address: &envoy_config_core_v3.Address_SocketAddress{ + SocketAddress: &envoy_config_core_v3.SocketAddress{ + Protocol: envoy_config_core_v3.SocketAddress_TCP, Address: upstreamServiceHost, - PortSpecifier: &core.SocketAddress_PortValue{ + PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{ PortValue: upstreamServicePort, }, }, @@ -215,26 +228,56 @@ func createLoadAssignment(clusterName string, upstreamServiceHost string, upstre }}, } + // TODO: Should be calling `ValidateAll()` here. + // if err := upstreamEndpoint.ValidateAll(); err != nil { + // return fmt.Errorf("EnvoyConfiguration.createLoadAssignment: failed to validate clusterLoadAssignment=%v, %w", upstreamEndpoint, err) + // } + return upstreamEndpoint } func (e *EnvoyConfiguration) GenerateSnapshot() (*cache.Snapshot, error) { - var clusters []cacheTypes.Resource - for _, c := range e.clusters { - clusters = append(clusters, c) + if err := e.listener.ValidateAll(); err != nil { + return nil, fmt.Errorf("EnvoyConfiguration.GenerateSnapshot: failed to validate listener=%v, %w", e.listener, err) } + + clusters := []envoy_cache_types.Resource{} + for _, cluster := range e.clusters { + if err := cluster.ValidateAll(); err != nil { + return nil, fmt.Errorf("EnvoyConfiguration.GenerateSnapshot: failed to validate cluster=%v, %w", cluster, err) + } + clusters = append(clusters, cluster) + } + + secrets := []*envoy_extensions_transport_sockets_tls_v3.Secret{} + for _, secret := range e.secrets { + if err := secret.ValidateAll(); err != nil { + return nil, fmt.Errorf("EnvoyConfiguration.GenerateSnapshot: failed to validate secret=%v, %w", secret, err) + } + secrets = append(secrets, secret) + } + // We're using uuid V1 to provide time sortable snapshot version - snapshotVersion, _ := uuid.NewV1() - snap, err := cache.NewSnapshot(snapshotVersion.String(), - map[resource.Type][]cacheTypes.Resource{ - resource.ClusterType: clusters, - resource.RouteType: {e.makeRouteConfiguration(RouteName)}, - resource.ListenerType: {e.listener}, + snapshotVersion, err := uuid.NewV1() + if err != nil { + return nil, fmt.Errorf("EnvoyConfiguration.GenerateSnapshot: failed on uuid.NewV1, %w", err) + } + + newVersion := snapshotVersion.String() + e.logger.Info("GenerateSnapshot: generating new snapshot", "newVersion", newVersion) + + snap, err := cache.NewSnapshot(newVersion, + map[envoy_resource_v3.Type][]envoy_cache_types.Resource{ + envoy_resource_v3.ClusterType: clusters, + envoy_resource_v3.RouteType: {e.makeRouteConfiguration(RouteName)}, + envoy_resource_v3.ListenerType: {e.listener}, + envoy_resource_v3.SecretType: asResources(secrets), }, ) if err != nil { - return nil, err + return nil, fmt.Errorf("EnvoyConfiguration.GenerateSnapshot: failed to create new snapshot, newVersion=%v, %w", newVersion, err) } + return snap, snap.Consistent() } @@ -250,6 +293,90 @@ func (e *EnvoyConfiguration) makeRouteConfiguration(routeConfigName string) *rou } } +// AddGenericSecretString +// Add a generic secrets as a string. This should be used to add the `token_secret`, also known as `TokenSecretType`. +func (e *EnvoyConfiguration) AddGenericSecretString(inlineString string) (string, error) { + const SecretType = types.TokenSecretType + + name, err := e.generateSecretName(SecretType) + if err != nil { + return "", fmt.Errorf("EnvoyConfiguration.AddGenericSecretBytes: failed to generate %q name, %w", SecretType, err) + } + secret := &envoy_extensions_transport_sockets_tls_v3.Secret{ + Name: name, + Type: &envoy_extensions_transport_sockets_tls_v3.Secret_GenericSecret{ + GenericSecret: &envoy_extensions_transport_sockets_tls_v3.GenericSecret{ + Secret: &envoy_config_core_v3.DataSource{ + Specifier: &envoy_config_core_v3.DataSource_InlineString{ + InlineString: inlineString, + }, + }, + }, + }, + } + return e.addGenericSecret(SecretType, secret) +} + +// AddGenericSecretBytes +// Add a generic secrets as a string. This should be used to add the `hmac_secret`, also known as `HMACSecretType`. +func (e *EnvoyConfiguration) AddGenericSecretBytes(inlineBytes []byte) (string, error) { + const SecretType = types.HMACSecretType + + name, err := e.generateSecretName(SecretType) + if err != nil { + return "", fmt.Errorf("EnvoyConfiguration.AddGenericSecretBytes: failed to generate %q name, %w", SecretType, err) + } + secret := &envoy_extensions_transport_sockets_tls_v3.Secret{ + Name: name, + Type: &envoy_extensions_transport_sockets_tls_v3.Secret_GenericSecret{ + GenericSecret: &envoy_extensions_transport_sockets_tls_v3.GenericSecret{ + Secret: &envoy_config_core_v3.DataSource{ + Specifier: &envoy_config_core_v3.DataSource_InlineBytes{ + InlineBytes: inlineBytes, + }, + }, + }, + }, + } + return e.addGenericSecret(SecretType, secret) +} + +// addGenericSecret +// The `name` field in the secret along with a uuid-v4 is used as the key to value in the map. +func (e *EnvoyConfiguration) addGenericSecret(secretType types.SecretType, secret *envoy_extensions_transport_sockets_tls_v3.Secret) (string, error) { + if secret == nil { + return "", fmt.Errorf("EnvoyConfiguration.addGenericSecret: %q is nil", secretType) + } + + if err := secret.ValidateAll(); err != nil { + return "", fmt.Errorf("EnvoyConfiguration.addGenericSecret: %q failed validation %v, %w", secretType, spew.Sprint(secret), err) + } + + key := secret.Name + e.secrets[key] = secret + e.logger.Info("AddClientSecret: added secret", "key", key, "secret", spew.Sprint(secret), "secretType", secretType) + + return key, nil +} + +// addGenericSecret +// The `name` field in the secret along with a uuid-v4 is used as the key to value in the map. +func (e *EnvoyConfiguration) generateSecretName(secretType types.SecretType) (string, error) { + uuid, err := google_uuid.NewRandom() + if err != nil { + return "", fmt.Errorf("EnvoyConfiguration.generateSecretName: failed to get UUID for %q, %w", secretType, err) + } + + keyPrefix := "" + if secretType == types.TokenSecretType { + keyPrefix = "token-secret" + } else if secretType == types.HMACSecretType { + keyPrefix = "hmac-secret" + } + + return fmt.Sprintf("%s-%s", keyPrefix, uuid.String()), nil +} + // sortRoutesByPathMatcher creates route list ordered by: // * path matcher // * regex path matcher, longest regex path first diff --git a/internal/envoy/config/hcm.go b/internal/envoy/config/hcm.go index 4d22293d6..d252e53fb 100644 --- a/internal/envoy/config/hcm.go +++ b/internal/envoy/config/hcm.go @@ -239,13 +239,11 @@ func ConfigSource(cluster string) *envoy_core_v3.ConfigSource { ResourceApiVersion: envoy_core_v3.ApiVersion_V3, ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_ApiConfigSource{ ApiConfigSource: &envoy_core_v3.ApiConfigSource{ - TransportApiVersion: envoy_core_v3.ApiVersion_V3, - ApiType: envoy_core_v3.ApiConfigSource_GRPC, - RequestTimeout: durationpb.New(defaultRequestTimeout), - SetNodeOnFirstMessageOnly: true, - GrpcServices: []*envoy_core_v3.GrpcService{ - makeGrpcService(cluster, "", defaultResponseTimeout), - }, + TransportApiVersion: envoy_core_v3.ApiVersion_V3, + ApiType: envoy_core_v3.ApiConfigSource_GRPC, + RequestTimeout: durationpb.New(defaultRequestTimeout), + GrpcServices: []*envoy_core_v3.GrpcService{makeGrpcService(cluster, "", defaultResponseTimeout)}, + //SetNodeOnFirstMessageOnly: true, // Could someone please justify why this is set to `true`? }, }, } diff --git a/internal/envoy/config/resource_cache_type_utils.go b/internal/envoy/config/resource_cache_type_utils.go new file mode 100644 index 000000000..b97369435 --- /dev/null +++ b/internal/envoy/config/resource_cache_type_utils.go @@ -0,0 +1,47 @@ +// MIT License +// +// Copyright (c) 2022 Kubeshop +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package config + +import ( + "reflect" + + cacheTypes "github.com/envoyproxy/go-control-plane/pkg/cache/types" +) + +// asResources casts the given slice of values (that implement the envoy_types.Resource +// interface) to a slice of envoy_types.Resource. If the length of the slice is 0, it +// returns nil. +func asResources(messages interface{}) []cacheTypes.Resource { + v := reflect.ValueOf(messages) + if v.Len() == 0 { + return nil + } + + protos := make([]cacheTypes.Resource, v.Len()) + + for i := range protos { + protos[i] = v.Index(i).Interface().(cacheTypes.Resource) + } + + return protos +} diff --git a/internal/envoy/manager/cache_manager.go b/internal/envoy/manager/cache_manager.go index ca0842bf8..eeeaf21ad 100644 --- a/internal/envoy/manager/cache_manager.go +++ b/internal/envoy/manager/cache_manager.go @@ -44,7 +44,7 @@ type cacheManager struct { func NewCacheManager(snapshotCache cache_v3.SnapshotCache, logger logr.Logger) *cacheManager { return &cacheManager{ SnapshotCache: snapshotCache, - fleetSnapshot: make(map[string]*cache_v3.Snapshot), + fleetSnapshot: map[string]*cache_v3.Snapshot{}, mu: sync.RWMutex{}, logger: logger.WithName("CacheManager"), } diff --git a/internal/envoy/manager/envoy_callbacks.go b/internal/envoy/manager/envoy_callbacks.go index fcaae9111..7f9361b2b 100644 --- a/internal/envoy/manager/envoy_callbacks.go +++ b/internal/envoy/manager/envoy_callbacks.go @@ -21,6 +21,8 @@ // SOFTWARE. // Callbacks are called by GRPC server on new events. +// +// `l.logger.V(1)` is effectively debug level. package manager import ( @@ -56,21 +58,11 @@ func (c *Callbacks) OnDeltaStreamOpen(ctx context.Context, id int64, typeUrl str } func (c *Callbacks) OnDeltaStreamClosed(id int64) { - // `l.logger.V(1)` is effectively debug level. c.logger.V(1).Info("OnDeltaStreamClosed", "id", id) } func (c *Callbacks) OnStreamRequest(id int64, request *envoy_discovery_v3.DiscoveryRequest) error { - c.logger.V(1).Info("OnStreamRequest", "id", id, "request.TypeUrl", request.TypeUrl) - if c.cacheManager.IsNodeExist(request.Node.Id) { - return nil - } - - if err := c.cacheManager.setNodeSnapshot(request.Node.Id, request.Node.Cluster); err != nil { - c.logger.Error(err, "OnStreamRequest", "id", id, "request.TypeUrl", request.TypeUrl) - return err - } - + c.logger.Info("OnStreamRequest", "id", id, "request.TypeUrl", request.TypeUrl, "request.Node.Cluster", request.Node.Cluster, "request.Node.Id", request.Node.Id) return nil } diff --git a/internal/envoy/manager/envoy_config_manager.go b/internal/envoy/manager/envoy_config_manager.go index c109b3902..74503409d 100644 --- a/internal/envoy/manager/envoy_config_manager.go +++ b/internal/envoy/manager/envoy_config_manager.go @@ -91,11 +91,7 @@ func registerServer(grpcServer *grpc.Server, server server.Server) { clusterservice.RegisterClusterDiscoveryServiceServer(grpcServer, server) routeservice.RegisterRouteDiscoveryServiceServer(grpcServer, server) listenerservice.RegisterListenerDiscoveryServiceServer(grpcServer, server) - secretservice.RegisterSecretDiscoveryServiceServer(grpcServer, server) - // TODO(MBana): Not too sure about this one, but I'm leaving it as a reference that an unimplemented SDS could be used. - // secretservice.RegisterSecretDiscoveryServiceServer(grpcServer, &secretservice.UnimplementedSecretDiscoveryServiceServer{}) - runtimeservice.RegisterRuntimeDiscoveryServiceServer(grpcServer, server) } diff --git a/internal/envoy/types/secrets.go b/internal/envoy/types/secrets.go new file mode 100644 index 000000000..2457fc0a1 --- /dev/null +++ b/internal/envoy/types/secrets.go @@ -0,0 +1,30 @@ +// MIT License +// +// Copyright (c) 2022 Kubeshop +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package types + +type SecretType string + +const ( + TokenSecretType SecretType = "token_secret" + HMACSecretType SecretType = "hmac_secret" +) diff --git a/pkg/options/auth.go b/pkg/options/auth.go index 2e1b9339b..2908849a7 100644 --- a/pkg/options/auth.go +++ b/pkg/options/auth.go @@ -168,7 +168,8 @@ type Credentials struct { ClientSecret string `json:"client_secret,omitempty" yaml:"client_secret,omitempty"` // REQUIRED. TokenSecret string `json:"token_secret,omitempty" yaml:"token_secret,omitempty"` - // REQUIRED. + // If not specified, `kusk-gateway` generates the `hmac_secret`. + // OPTIONAL. HmacSecret string `json:"hmac_secret,omitempty" yaml:"hmac_secret,omitempty"` // OPTIONAL. CookieNames CookieNames `json:"cookie_names,omitempty" yaml:"cookie_names,omitempty"` @@ -186,10 +187,13 @@ func (o Credentials) Validate() error { // CookieNames - By default, OAuth2 filter sets some cookies with the following names: BearerToken, OauthHMAC, and OauthExpires. These cookie names can be customized by setting cookie_names. type CookieNames struct { // Defaults to BearerToken. + // OPTIONAL. BearerToken string `json:"bearer_token,omitempty" yaml:"bearer_token,omitempty"` // Defaults to OauthHMAC. + // OPTIONAL. OauthHMAC string `json:"oauth_hmac,omitempty" yaml:"oauth_hmac,omitempty"` // Defaults to OauthExpires. + // OPTIONAL. ExpiresOauth string `json:"oauth_expires,omitempty" yaml:"oauth_expires,omitempty"` } diff --git a/smoketests/Makefile b/smoketests/Makefile index c5d99c12c..72d748fcd 100644 --- a/smoketests/Makefile +++ b/smoketests/Makefile @@ -28,8 +28,11 @@ check-openapi-path: check-auth_oauth2: kubectl apply -f ../examples/auth/oauth2/authorization-code-grant/manifests.yaml kubectl wait deployment --namespace default auth-oauth2-oauth0-authorization-code-grant-go-httpbin --for condition=Available=True --timeout=3m + kubectl port-forward --namespace default deployments/default 19000:19000 & echo $$! > /tmp/kube-port-forward-check-auth_oauth2.pid + sleep 2 go test -count=1 -v github.com/kubeshop/kusk-gateway/smoketests/$(subst check-,,$@) kubectl delete -f ../examples/auth/oauth2/authorization-code-grant/manifests.yaml + bash -c "trap 'pkill -F /tmp/kube-port-forward-check-auth_oauth2.pid' SIGINT SIGTERM ERR EXIT" sandbox: @docker build samples/hello-world/hello-world-container/ -t localhost:50000/hello-world:smoke diff --git a/smoketests/auth_oauth2/auth_oauth2_test.go b/smoketests/auth_oauth2/auth_oauth2_test.go index 436aa6e52..94f15379d 100644 --- a/smoketests/auth_oauth2/auth_oauth2_test.go +++ b/smoketests/auth_oauth2/auth_oauth2_test.go @@ -28,7 +28,9 @@ package auth_oauth2 import ( "context" + "encoding/json" "fmt" + "io" "net/http" "strings" "testing" @@ -83,7 +85,7 @@ func (t *AuthOAuth2TestSuite) SetupTest() { t.api = api // store `api` for deletion later - duration := 5 * time.Second + duration := 4 * time.Second t.T().Logf("Sleeping for %s", duration) time.Sleep(duration) // weird way to wait it out probably needs to be done dynamically } @@ -138,6 +140,45 @@ func (t *AuthOAuth2TestSuite) TestRootPathReturnsARedirect() { t.Contains(redirect.String(), redirectExpected) } +func (t *AuthOAuth2TestSuite) TestSecretsAreDynamicActiveSecrets() { + t.T().Skipf("Skipping %s due to potential Envoy bug", t.T().Name()) + + const ExpectedDynamicActiveSecretsCount = 4 + + // 1. Secrets should appear as `dynamic_active_secrets` and not as `dynamic_warming_secrets` in the admin `/config_dump` page. + // 2. There should be a total of 4 secrets. + + url := "http://localhost:19000/" + request, err := http.NewRequest(http.MethodGet, url, nil) + t.NoError(err) + + client := makeHTTPClient() + response, err := client.Do(request) + t.NoError(err) + t.Equal(http.StatusOK, response.StatusCode) + + defer func() { + t.NoError(response.Body.Close()) + }() + + responseBody, err := io.ReadAll(response.Body) + t.NoError(err) + + body := map[string][]interface{}{} + t.NoError(json.Unmarshal(responseBody, &body)) + + configs := body["configs"] + t.Len(configs, 1) + + config := configs[0] + t.T().Logf("config=%+v\n", config) + + // Ensure that all secrets appear under `dynamic_active_secrets` and that there are no `dynamic_warming_secrets` + // TODO(MBana): I've intentionally left this code commented out. + // dynamicActiveSecrets := []interface{}{} + // t.Len(dynamicActiveSecrets, ExpectedDynamicActiveSecretsCount) +} + func getEnvoyFleetSvc(t *common.KuskTestSuite) *corev1.Service { t.T().Helper()