OAuth2: Dynamic Secrets

Support defining the secrets dynamically, i.e., they should appear as `dynamic_active_secrets` in the admin `/config_dump` page:

```sh
$ kubectl port-forward --namespace default deployment/default 19000:19000 &
$ curl -s 'http://localhost:19000/stats' | grep sds
sds.client_secret_test_1.version_text: "690"
sds.hmac_secret_test_1.version_text: "690"
cluster.kubeshop-kusk-gateway-oauth2.eu.auth0.com.client_ssl_socket_factory.ssl_context_update_by_sds: 0
sds.client_secret_test_1.init_fetch_timeout: 0
sds.client_secret_test_1.key_rotation_failed: 0
sds.client_secret_test_1.update_attempt: 2
sds.client_secret_test_1.update_failure: 0
sds.client_secret_test_1.update_rejected: 0
sds.client_secret_test_1.update_success: 1
sds.client_secret_test_1.update_time: 1662408207693
sds.client_secret_test_1.version: 18219575566587264222
sds.hmac_secret_test_1.init_fetch_timeout: 0
sds.hmac_secret_test_1.key_rotation_failed: 0
sds.hmac_secret_test_1.update_attempt: 2
sds.hmac_secret_test_1.update_failure: 0
sds.hmac_secret_test_1.update_rejected: 0
sds.hmac_secret_test_1.update_success: 1
sds.hmac_secret_test_1.update_time: 1662408207693
sds.hmac_secret_test_1.version: 18219575566587264222
sds.client_secret_test_1.update_duration: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.5(nan,0) P99.9(nan,0) P100(nan,0)
sds.hmac_secret_test_1.update_duration: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.5(nan,0) P99.9(nan,0) P100(nan,0)
$ curl -s 'http://localhost:19000/config_dump' | jq '.[] | .[-1].dynamic_active_secrets'
[
  {
    "name": "client_secret_test_1",
    "version_info": "690",
    "last_updated": "2022-09-05T20:03:27.693Z",
    "secret": {
      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
      "name": "client_secret_test_1",
      "generic_secret": {
        "secret": {
          "inline_string": "[redacted]"
        }
      }
    }
  },
  {
    "name": "hmac_secret_test_1",
    "version_info": "690",
    "last_updated": "2022-09-05T20:03:27.693Z",
    "secret": {
      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
      "name": "hmac_secret_test_1",
      "generic_secret": {
        "secret": {
          "inline_bytes": "W3JlZGFjdGVkXQ=="
        }
      }
    }
  }
]
```

Things to note from the above output are:

1. The secrets are in the `dynamic_active_secrets` object not in a `dynamic_warming_secrets` object, the later of which would mean that the secrets have not been sent to envoy. See <https://www.envoyproxy.io/docs/envoy/v1.23.1/api-v3/admin/v3/config_dump.proto#admin-v3-secretsconfigdump> for more information. As per the documentation `dynamic_active_secrets`:

> "The dynamically loaded active secrets. These are secrets that are available to service clusters or listeners."

2. The output from the `/stats` endpoint shows no failures.

Signed-off-by: Mohamed Bana <mohamed@bana.io>

Makefile

@@ -80,20 +80,20 @@ tail-envoyfleet: ## Tail logs of envoy
 .PHONY: enable-logging
 enable-logging: ## Set some particular logger's level
 	kubectl port-forward --namespace default deployments/default 19000:19000 & echo $$! > /tmp/kube-port-forward-logging.pid
-	sleep 4
-	curl -s -X POST "http://localhost:19000/logging?backtrace=trace"
-	curl -s -X POST "http://localhost:19000/logging?envoy_bug=trace"
-	curl -s -X POST "http://localhost:19000/logging?assert=trace"
-	curl -s -X POST "http://localhost:19000/logging?secret=trace"
-	curl -s -X POST "http://localhost:19000/logging?grpc=trace"
-	curl -s -X POST "http://localhost:19000/logging?ext_authz=trace"
-	curl -s -X POST "http://localhost:19000/logging?filter=trace"
-	curl -s -X POST "http://localhost:19000/logging?misc=trace"
-	curl -s -X POST "http://localhost:19000/logging?conn_handler=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?connection=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?http=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?http2=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?admin=trace"
+	sleep 2
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?backtrace=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?envoy_bug=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?assert=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?secret=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?grpc=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?ext_authz=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?filter=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?misc=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?conn_handler=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?connection=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http2=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?admin=trace"
 	@# bash -c "trap 'pkill -F /tmp/kube-port-forward-logging.pid' SIGINT SIGTERM ERR EXIT"
 	@echo
 	@echo "How to stop port forward to the admin port (19000):"

cmd/manager/main.go

@@ -267,15 +267,16 @@ func main() {
 	}()
 
 	secretsChan := make(chan *corev1.Secret)
-	controllerConfigManager := controllers.KubeEnvoyConfigManager{
-		Client:             mgr.GetClient(),
-		Scheme:             mgr.GetScheme(),
-		EnvoyManager:       envoyManager,
-		Validator:          proxy,
-		SecretToEnvoyFleet: map[string]gateway.EnvoyFleetID{},
-		WatchedSecretsChan: secretsChan,
-		OpenApiParser:      spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}),
-	}
+
+	controllerConfigManager := controllers.NewKubeEnvoyConfigManager(
+		mgr.GetClient(),
+		mgr.GetScheme(),
+		envoyManager,
+		proxy,
+		secretsChan,
+		map[string]gateway.EnvoyFleetID{},
+		spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}),
+	)
 
 	analytics.SendAnonymousInfo(ctx, controllerConfigManager.Client, "kusk", "kusk-gateway manager bootstrapping")
 	heartBeat(ctx, controllerConfigManager.Client)
@@ -294,7 +295,7 @@ func main() {
 	if err = (&controllers.EnvoyFleetReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "EnvoyFleet").
@@ -316,7 +317,7 @@ func main() {
 	if err = (&controllers.APIReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "API").
@@ -332,7 +333,7 @@ func main() {
 	if err = (&controllers.StaticRouteReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "StaticRoute").

config/samples/gateway_v1_envoyfleet.yaml

@@ -78,7 +78,7 @@ spec:
     # The output example:
     # "[2021-12-15T16:50:50.217Z]" "GET" "/" "200" "1"
     text_template: |
-      "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%"
+      "[%START_TIME%]" "%REQ(:scheme)%" "%REQ(:METHOD)%" "%REQ(X-FORWARDED-PROTO)%://%REQ(:AUTHORITY)%%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" %BYTES_SENT% %DURATION% %UPSTREAM_HOST% %REQ(X-ENVOY-IP-TAGS)%"
     # Json format fields order isn't preserved
     # The output example:
     # {"start_time":"2021-12-15T16:46:52.135Z","path":"/","response_code":200,"method":"GET","duration":1}

docs/docs/extension.md

@@ -327,7 +327,7 @@ The `auth` object contains the following properties to configure HTTP authentica
 | `auth.oauth2.authorization_endpoint`    | **Required, if `scheme` is `oauth2`**. Defines the `authorization_endpoint`, e.g., the field `authorization_endpoint` from <https://kubeshop-kusk-gateway-oauth2.eu.auth0.com/.well-known/openid-configuration>. |
 | `auth.oauth2.credentials.client_id`     | **Required, if `scheme` is `oauth2`**. Defines the Client ID. |
 | `auth.oauth2.credentials.client_secret` | **Required, if `scheme` is `oauth2`**. Defines the Client Secret. |
-| `auth.oauth2.redirect_uri`              | **Required, if `scheme` is `oauth2`**. The redirect URI passed to the authorization endpoint. |
+| `auth.oauth2.redirect_uri`              | **Required, if `scheme` is `oauth2`**. The redirect URI passed to the authorization endpoint. It is advised to use `%REQ(x-forwarded-proto)%://%REQ(:authority)%` before the actual callback path, e.g., the `redirect_uri` should be defined as `""%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback/oauth2/callback"`, including the quotes. |
 | `auth.oauth2.signout_path`              | **Required, if `scheme` is `oauth2`**. The path to sign a user out, clearing their credential cookies. |
 | `auth.oauth2.redirect_path_matcher`     | **Required, if `scheme` is `oauth2`**. After a redirecting the user back to the `redirect_uri`, using this new grant and the `token_secret`, the `kusk-gateway` then attempts to retrieve an access token from the `token_endpoint`. The `kusk-gateway` knows it has to do this instead of reinitiating another login because the incoming request has a path that matches the `redirect_path_matcher` criteria. |
 | `auth.oauth2.forward_bearer_token`      | **Required, if `scheme` is `oauth2`**. If the Bearer Token should be forwarded, you generally want this to be `true`. When the authn server validates the client and returns an authorization token back to `kusk-gateway`, no matter what format that token is, if `forward_bearer_token` is set to true `kusk-gateway` will send over a cookie named `BearerToken` to the upstream. Additionally, the `Authorization` header will be populated with the same value, i.e., Forward the OAuth token as a Bearer to upstream web service. |
@@ -391,7 +391,7 @@ x-kusk:
       credentials:
         client_id: *CLIENT_ID*
         client_secret: *CLIENT_SECRET*
-      redirect_uri: /oauth2/callback
+      redirect_uri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback"
       redirect_path_matcher: /oauth2/callback
       signout_path: /oauth2/signout
       forward_bearer_token: true

examples/auth/oauth2/authorization-code-grant/api.yaml

@@ -30,20 +30,21 @@ spec:
           credentials:
             client_id: upRN78W8GzV4TwFRp0ekZfLx2UnqJJs8
             client_secret: Z6MX7NreJumWLmf6unsQ5uiEUrTBxfNtqG9Vy5Kjktnvfj-_fRCBO9EU1mL1YzAJ
-          redirect_uri: /oauth2/callback
+          #redirect_uri: "%REQ(:scheme)%://%REQ(:authority)%/oauth2/callback"
+          redirect_uri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/oauth2/callback"
           redirect_path_matcher: /oauth2/callback
           signout_path: /oauth2/signout
           forward_bearer_token: true
           auth_scopes:
             - openid
     paths:
-      "/":
-        get:
-          description: Returns GET data.
-          operationId: "/get"
-          responses: {}
       "/uuid":
         get:
           description: Returns UUID4.
           operationId: "/uuid"
           responses: {}
+      # "/":
+      #   get:
+      #     description: Returns GET data.
+      #     operationId: "/get"
+      #     responses: {}

go.mod

@@ -90,7 +90,7 @@ require (
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.3.0
 	github.com/gookit/color v1.5.2
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect

internal/controllers/config_manager.go

@@ -54,15 +54,26 @@ const (
 // KubeEnvoyConfigManager manages all Envoy configurations parsing from CRDs
 type KubeEnvoyConfigManager struct {
 	client.Client
-	Scheme       *runtime.Scheme
-	EnvoyManager *manager.EnvoyConfigManager
-	Validator    validation.ValidationUpdater
-	m            sync.Mutex
-
+	Scheme             *runtime.Scheme
+	EnvoyManager       *manager.EnvoyConfigManager
+	Validator          validation.ValidationUpdater
 	WatchedSecretsChan chan *v1.Secret
 	SecretToEnvoyFleet map[string]gateway.EnvoyFleetID
+	OpenApiParser      spec.Parser
+	m                  *sync.Mutex
+}
 
-	OpenApiParser spec.Parser
+func NewKubeEnvoyConfigManager(client client.Client, scheme *runtime.Scheme, envoyManager *manager.EnvoyConfigManager, validator validation.ValidationUpdater, watchedSecretsChan chan *v1.Secret, secretToEnvoyFleet map[string]gateway.EnvoyFleetID, OpenApiParser spec.Parser) *KubeEnvoyConfigManager {
+	return &KubeEnvoyConfigManager{
+		Client:             client,
+		Scheme:             scheme,
+		EnvoyManager:       envoyManager,
+		Validator:          validator,
+		WatchedSecretsChan: watchedSecretsChan,
+		SecretToEnvoyFleet: secretToEnvoyFleet,
+		OpenApiParser:      OpenApiParser,
+		m:                  &sync.Mutex{},
+	}
 }
 
 var (
@@ -71,7 +82,6 @@ var (
 
 // UpdateConfiguration is the main method to gather all routing configs and to create and apply Envoy config
 func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetID gateway.EnvoyFleetID) error {
-
 	l := configManagerLogger
 	fleetIDstr := fleetID.String()
 	// acquiring this lock is required so that no potentially conflicting updates would happen at the same time
@@ -82,7 +92,8 @@ func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetI
 	l.Info("Started updating configuration", "fleet", fleetIDstr)
 	defer l.Info("Finished updating configuration", "fleet", fleetIDstr)
 
-	envoyConfig := config.New()
+	envoyConfig := config.NewEnvoyConfiguration(ctrl.Log)
+
 	// fetch all APIs and Static Routes to rebuild Envoy configuration
 	l.Info("Getting APIs for the fleet", "fleet", fleetIDstr)
 

internal/controllers/envoy.yaml

@@ -16,23 +16,14 @@ dynamic_resources:
     ads: {}
 
 static_resources:
-  secrets:
-    - name: token
-      generic_secret:
-        secret:
-          inline_string: "<stub_token_secret>"
-    - name: hmac
-      generic_secret:
-        secret:
-          inline_bytes: "KVPwEwpMmxmqxvT+tZu07vCDwpT41/vnjeM5kLW78Vc="
   clusters:
-    - type: STRICT_DNS
+    - name: xds_cluster
+      type: STRICT_DNS
       typed_extension_protocol_options:
         envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
           "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
           explicit_http_config:
             http2_protocol_options: {}
-      name: xds_cluster
       load_assignment:
         cluster_name: xds_cluster
         endpoints:

internal/envoy/auth/oauth2_filter.go

@@ -28,13 +28,16 @@ import (
 	"fmt"
 	"net/url"
 	"strconv"
+	"time"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_extensions_filter_http_oauth2_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/oauth2/v3"
 	envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 
 	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/durationpb"
 
+	"github.com/kubeshop/kusk-gateway/internal/envoy/types"
 	"github.com/kubeshop/kusk-gateway/pkg/options"
 )
 
@@ -81,15 +84,35 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr
 	}
 	authorizationEndpoint := oauth2Options.AuthorizationEndpoint
 
+	sdsConfig := makeConfigSource()
+
+	tokenSecretName, err := args.EnvoyConfiguration.AddGenericSecretString(oauth2Options.Credentials.ClientSecret)
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add %q, %w", types.TokenSecretType, err)
+	}
+
 	tokenSecret := &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{
-		Name: "token",
+		Name:      tokenSecretName,
+		SdsConfig: sdsConfig,
+	}
+
+	hmac, err := GenerateHMAC()
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: cannot generate HMAC, %w", err)
+	}
+
+	hmacSecretName, err := args.EnvoyConfiguration.AddGenericSecretBytes([]byte(hmac))
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add %q, %w", types.HMACSecretType, err)
 	}
 
 	tokenFormation := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials_HmacSecret{
 		HmacSecret: &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{
-			Name: "hmac",
+			Name:      hmacSecretName,
+			SdsConfig: sdsConfig,
 		},
 	}
+
 	credentials := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials{
 		// The client_id to be used in the authorize calls. This value will be URL encoded when sent to the OAuth server.
 		ClientId: oauth2Options.Credentials.ClientID,
@@ -102,8 +125,10 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr
 		TokenFormation: tokenFormation,
 	}
 
+	// // For example, `redirectUri` becomes "http://192.168.49.2/oauth2/callback"
+	// redirectUri := fmt.Sprintf("%s://%s%s", "%REQ(x-forwarded-proto)%", "%REQ(:authority)%", oauth2Options.RedirectURI)
 	// For example, `redirectUri` becomes "http://192.168.49.2/oauth2/callback"
-	redirectUri := fmt.Sprintf("%s://%s%s", "%REQ(x-forwarded-proto)%", "%REQ(:authority)%", oauth2Options.RedirectURI)
+	redirectUri := oauth2Options.RedirectURI
 	redirectPathMatcher := PathMatcherExact(oauth2Options.RedirectPathMatcher, false)
 	signoutPath := PathMatcherExact(oauth2Options.SignoutPath, false)
 	forwardBearerToken := oauth2Options.ForwardBearerToken
@@ -156,6 +181,16 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr
 	return anyOAuth2, nil
 }
 
+func makeConfigSource() *envoy_config_core_v3.ConfigSource {
+	return &envoy_config_core_v3.ConfigSource{
+		ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{
+			Ads: &envoy_config_core_v3.AggregatedConfigSource{},
+		},
+		InitialFetchTimeout: durationpb.New(time.Second * 128),
+		ResourceApiVersion:  ResourceApiVersion,
+	}
+}
+
 func GenerateHMAC() (string, error) {
 	// Since HMAC use symmetric key algorithm, we can just generate random bytes as secret key.