OAuth2: Dynamic Secrets

Support defining the secrets dynamically, i.e., they should appear as `dynamic_active_secrets` in the admin `/config_dump` page:

```sh
$ kubectl port-forward --namespace default deployment/default 19000:19000 &
$ curl -s 'http://localhost:19000/stats' | grep sds
sds.client_secret_test_1.version_text: "690"
sds.hmac_secret_test_1.version_text: "690"
cluster.kubeshop-kusk-gateway-oauth2.eu.auth0.com.client_ssl_socket_factory.ssl_context_update_by_sds: 0
sds.client_secret_test_1.init_fetch_timeout: 0
sds.client_secret_test_1.key_rotation_failed: 0
sds.client_secret_test_1.update_attempt: 2
sds.client_secret_test_1.update_failure: 0
sds.client_secret_test_1.update_rejected: 0
sds.client_secret_test_1.update_success: 1
sds.client_secret_test_1.update_time: 1662408207693
sds.client_secret_test_1.version: 18219575566587264222
sds.hmac_secret_test_1.init_fetch_timeout: 0
sds.hmac_secret_test_1.key_rotation_failed: 0
sds.hmac_secret_test_1.update_attempt: 2
sds.hmac_secret_test_1.update_failure: 0
sds.hmac_secret_test_1.update_rejected: 0
sds.hmac_secret_test_1.update_success: 1
sds.hmac_secret_test_1.update_time: 1662408207693
sds.hmac_secret_test_1.version: 18219575566587264222
sds.client_secret_test_1.update_duration: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.5(nan,0) P99.9(nan,0) P100(nan,0)
sds.hmac_secret_test_1.update_duration: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.5(nan,0) P99.9(nan,0) P100(nan,0)
$ curl -s 'http://localhost:19000/config_dump' | jq '.[] | .[-1].dynamic_active_secrets'
[
  {
    "name": "client_secret_test_1",
    "version_info": "690",
    "last_updated": "2022-09-05T20:03:27.693Z",
    "secret": {
      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
      "name": "client_secret_test_1",
      "generic_secret": {
        "secret": {
          "inline_string": "[redacted]"
        }
      }
    }
  },
  {
    "name": "hmac_secret_test_1",
    "version_info": "690",
    "last_updated": "2022-09-05T20:03:27.693Z",
    "secret": {
      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
      "name": "hmac_secret_test_1",
      "generic_secret": {
        "secret": {
          "inline_bytes": "W3JlZGFjdGVkXQ=="
        }
      }
    }
  }
]
```

Things to note from the above output are:

1. The secrets are in the `dynamic_active_secrets` object not in a `dynamic_warming_secrets` object, the later of which would mean that the secrets have not been sent to envoy. See <https://www.envoyproxy.io/docs/envoy/v1.23.1/api-v3/admin/v3/config_dump.proto#admin-v3-secretsconfigdump> for more information. As per the documentation `dynamic_active_secrets`:

> "The dynamically loaded active secrets. These are secrets that are available to service clusters or listeners."

2. The output from the `/stats` endpoint shows no failures.

Signed-off-by: Mohamed Bana <mohamed@bana.io>

Makefile

@@ -95,20 +95,20 @@ tail-envoyfleet: ## Tail logs of envoy
 .PHONY: enable-logging
 enable-logging: ## Set some particular logger's level
 	kubectl port-forward --namespace default deployments/default 19000:19000 & echo $$! > /tmp/kube-port-forward-logging.pid
-	sleep 4
-	curl -s -X POST "http://localhost:19000/logging?backtrace=trace"
-	curl -s -X POST "http://localhost:19000/logging?envoy_bug=trace"
-	curl -s -X POST "http://localhost:19000/logging?assert=trace"
-	curl -s -X POST "http://localhost:19000/logging?secret=trace"
-	curl -s -X POST "http://localhost:19000/logging?grpc=trace"
-	curl -s -X POST "http://localhost:19000/logging?ext_authz=trace"
-	curl -s -X POST "http://localhost:19000/logging?filter=trace"
-	curl -s -X POST "http://localhost:19000/logging?misc=trace"
-	curl -s -X POST "http://localhost:19000/logging?conn_handler=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?connection=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?http=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?http2=trace"
-	@# curl -s -X POST "http://localhost:19000/logging?admin=trace"
+	sleep 2
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?backtrace=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?envoy_bug=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?assert=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?secret=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?grpc=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?ext_authz=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?filter=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?misc=trace"
+	curl --silent --output /dev/null -X POST "http://localhost:19000/logging?conn_handler=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?connection=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?http2=trace"
+	@# curl --silent --output /dev/null -X POST "http://localhost:19000/logging?admin=trace"
 	@# bash -c "trap 'pkill -F /tmp/kube-port-forward-logging.pid' SIGINT SIGTERM ERR EXIT"
 	@echo
 	@echo "How to stop port forward to the admin port (19000):"

cmd/manager/main.go

@@ -265,14 +265,15 @@ func main() {
 	}()
 
 	secretsChan := make(chan *corev1.Secret)
-	controllerConfigManager := controllers.KubeEnvoyConfigManager{
-		Client:             mgr.GetClient(),
-		Scheme:             mgr.GetScheme(),
-		EnvoyManager:       envoyManager,
-		Validator:          proxy,
-		SecretToEnvoyFleet: map[string]gateway.EnvoyFleetID{},
-		WatchedSecretsChan: secretsChan,
-	}
+	controllerConfigManager := controllers.NewKubeEnvoyConfigManager(
+		mgr.GetClient(),
+		mgr.GetScheme(),
+		envoyManager,
+		proxy,
+		secretsChan,
+		map[string]gateway.EnvoyFleetID{},
+	)
+
 	analytics.SendAnonymousInfo(ctx, controllerConfigManager.Client, "kusk", "kusk-gateway manager bootstrapping")
 	heartBeat(ctx, controllerConfigManager.Client)
 
@@ -290,7 +291,7 @@ func main() {
 	if err = (&controllers.EnvoyFleetReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "EnvoyFleet").
@@ -312,7 +313,7 @@ func main() {
 	if err = (&controllers.APIReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "API").
@@ -328,7 +329,7 @@ func main() {
 	if err = (&controllers.StaticRouteReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
-		ConfigManager: &controllerConfigManager,
+		ConfigManager: controllerConfigManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.
 			WithValues("controller", "StaticRoute").

internal/controllers/config_manager.go

@@ -54,13 +54,24 @@ const (
 // KubeEnvoyConfigManager manages all Envoy configurations parsing from CRDs
 type KubeEnvoyConfigManager struct {
 	client.Client
-	Scheme       *runtime.Scheme
-	EnvoyManager *manager.EnvoyConfigManager
-	Validator    validation.ValidationUpdater
-	m            sync.Mutex
-
+	Scheme             *runtime.Scheme
+	EnvoyManager       *manager.EnvoyConfigManager
+	Validator          validation.ValidationUpdater
 	WatchedSecretsChan chan *v1.Secret
 	SecretToEnvoyFleet map[string]gateway.EnvoyFleetID
+	m                  *sync.Mutex
+}
+
+func NewKubeEnvoyConfigManager(client client.Client, scheme *runtime.Scheme, envoyManager *manager.EnvoyConfigManager, validator validation.ValidationUpdater, watchedSecretsChan chan *v1.Secret, secretToEnvoyFleet map[string]gateway.EnvoyFleetID) *KubeEnvoyConfigManager {
+	return &KubeEnvoyConfigManager{
+		Client:             client,
+		Scheme:             scheme,
+		EnvoyManager:       envoyManager,
+		Validator:          validator,
+		WatchedSecretsChan: watchedSecretsChan,
+		SecretToEnvoyFleet: secretToEnvoyFleet,
+		m:                  &sync.Mutex{},
+	}
 }
 
 var (
@@ -69,7 +80,6 @@ var (
 
 // UpdateConfiguration is the main method to gather all routing configs and to create and apply Envoy config
 func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetID gateway.EnvoyFleetID) error {
-
 	l := configManagerLogger
 	fleetIDstr := fleetID.String()
 	// acquiring this lock is required so that no potentially conflicting updates would happen at the same time
@@ -80,7 +90,7 @@ func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetI
 	l.Info("Started updating configuration", "fleet", fleetIDstr)
 	defer l.Info("Finished updating configuration", "fleet", fleetIDstr)
 
-	envoyConfig := config.New()
+	envoyConfig := config.NewEnvoyConfiguration(ctrl.Log)
 
 	// fetch all APIs and Static Routes to rebuild Envoy configuration
 	l.Info("Getting APIs for the fleet", "fleet", fleetIDstr)

internal/controllers/envoy.yaml

@@ -16,23 +16,14 @@ dynamic_resources:
     ads: {}
 
 static_resources:
-  secrets:
-    - name: token
-      generic_secret:
-        secret:
-          inline_string: "<stub_token_secret>"
-    - name: hmac
-      generic_secret:
-        secret:
-          inline_bytes: "KVPwEwpMmxmqxvT+tZu07vCDwpT41/vnjeM5kLW78Vc="
   clusters:
-    - type: STRICT_DNS
+    - name: xds_cluster
+      type: STRICT_DNS
       typed_extension_protocol_options:
         envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
           "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
           explicit_http_config:
             http2_protocol_options: {}
-      name: xds_cluster
       load_assignment:
         cluster_name: xds_cluster
         endpoints:

internal/envoy/auth/oauth2_filter.go

@@ -81,15 +81,35 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr
 	}
 	authorizationEndpoint := oauth2Options.AuthorizationEndpoint
 
+	sdsConfig := makeConfigSourceSpecifier()
+
 	tokenSecret := &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{
-		Name: "token",
+		Name:      "client_secret_test_1",
+		SdsConfig: sdsConfig,
+	}
+
+	args.EnvoyConfiguration.AddClientSecret("client_secret_test_1", oauth2Options.Credentials.ClientSecret)
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add `client_secret`, %w", err)
 	}
 
 	tokenFormation := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials_HmacSecret{
 		HmacSecret: &envoy_extensions_transport_sockets_tls_v3.SdsSecretConfig{
-			Name: "hmac",
+			Name:      "hmac_secret_test_1",
+			SdsConfig: sdsConfig,
 		},
 	}
+
+	hmac, err := GenerateHMAC()
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: cannot generate HMAC, %w", err)
+	}
+
+	args.EnvoyConfiguration.AddHMACSecret("hmac_secret_test_1", []byte(hmac))
+	if err != nil {
+		return nil, fmt.Errorf("auth.NewFilterHTTPOAuth2: failed to add `HmacSecret`, %w", err)
+	}
+
 	credentials := &envoy_extensions_filter_http_oauth2_v3.OAuth2Credentials{
 		// The client_id to be used in the authorize calls. This value will be URL encoded when sent to the OAuth server.
 		ClientId: oauth2Options.Credentials.ClientID,
@@ -156,6 +176,15 @@ func NewFilterHTTPOAuth2(oauth2Options *options.OAuth2, args *parseAuthOptionsAr
 	return anyOAuth2, nil
 }
 
+func makeConfigSourceSpecifier() *envoy_config_core_v3.ConfigSource {
+	return &envoy_config_core_v3.ConfigSource{
+		ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{
+			Ads: &envoy_config_core_v3.AggregatedConfigSource{},
+		},
+		ResourceApiVersion: ResourceApiVersion,
+	}
+}
+
 func GenerateHMAC() (string, error) {
 	// Since HMAC use symmetric key algorithm, we can just generate random bytes as secret key.