From 166b4d01657a15b59c994d7a1ae7dd00e9d8291a Mon Sep 17 00:00:00 2001 From: Dimitrij Drus Date: Thu, 2 May 2024 13:01:09 +0200 Subject: [PATCH 1/4] first working (not optimal) version with envoy gateway --- examples/kubernetes/Justfile | 92 ++++++++++--------- examples/kubernetes/cert-manager/ca.yaml | 24 +++++ .../cert-manager/cluster_issuer.yaml | 6 ++ examples/kubernetes/emissary/listener.yaml | 12 +++ examples/kubernetes/envoygw/gateway.yaml | 53 +++++++++++ .../demo-app/base/certificate.yaml | 2 +- .../http-proxy.yaml | 0 .../kustomization.yaml | 0 .../{emissary-ingress => emissary}/host.yaml | 0 .../kustomization.yaml | 0 .../mapping.yaml | 0 .../demo-app/overlays/envoygw/http_route.yaml | 22 +++++ .../overlays/envoygw/kustomization.yaml | 5 + .../{haproxy-ingress => haproxy}/ingress.yaml | 0 .../kustomization.yaml | 0 .../heimdall/backend-tls-policy.yaml | 18 ++++ .../quickstarts/heimdall/certificate.yaml | 20 +++- .../heimdall/envoygw-security-policy.yaml | 19 ++++ .../quickstarts/heimdall/helm-values.yaml | 4 + .../quickstarts/heimdall/namespace.yaml | 4 + .../quickstarts/heimdall/reference-grant.yaml | 44 +++++++++ 21 files changed, 279 insertions(+), 46 deletions(-) create mode 100644 examples/kubernetes/cert-manager/ca.yaml create mode 100644 examples/kubernetes/cert-manager/cluster_issuer.yaml create mode 100644 examples/kubernetes/emissary/listener.yaml create mode 100644 examples/kubernetes/envoygw/gateway.yaml rename examples/kubernetes/quickstarts/demo-app/overlays/{contour-ingress => contour}/http-proxy.yaml (100%) rename examples/kubernetes/quickstarts/demo-app/overlays/{contour-ingress => contour}/kustomization.yaml (100%) rename examples/kubernetes/quickstarts/demo-app/overlays/{emissary-ingress => emissary}/host.yaml (100%) rename examples/kubernetes/quickstarts/demo-app/overlays/{emissary-ingress => emissary}/kustomization.yaml (100%) rename examples/kubernetes/quickstarts/demo-app/overlays/{emissary-ingress => emissary}/mapping.yaml (100%) create mode 100644 examples/kubernetes/quickstarts/demo-app/overlays/envoygw/http_route.yaml create mode 100644 examples/kubernetes/quickstarts/demo-app/overlays/envoygw/kustomization.yaml rename examples/kubernetes/quickstarts/demo-app/overlays/{haproxy-ingress => haproxy}/ingress.yaml (100%) rename examples/kubernetes/quickstarts/demo-app/overlays/{haproxy-ingress => haproxy}/kustomization.yaml (100%) create mode 100644 examples/kubernetes/quickstarts/heimdall/backend-tls-policy.yaml create mode 100644 examples/kubernetes/quickstarts/heimdall/envoygw-security-policy.yaml create mode 100644 examples/kubernetes/quickstarts/heimdall/namespace.yaml create mode 100644 examples/kubernetes/quickstarts/heimdall/reference-grant.yaml diff --git a/examples/kubernetes/Justfile b/examples/kubernetes/Justfile index 03dfdd3a9..73e27d41f 100644 --- a/examples/kubernetes/Justfile +++ b/examples/kubernetes/Justfile @@ -7,11 +7,13 @@ nginx_version := '9.7.7' contour_version := '17.0.0' emissary_version := '8.7.2' haproxy_version := '0.14.4' +envoy_gw_version := 'v1.0.1' metallb_version := '0.13.10' -certmanager_version := '1.12.3' +certmanager_version := '1.14.5' +trustmanager_version := '0.9.2' cluster_name := 'demo-cluster' -default_ingress_controller := "contour" +default_router := "contour" setup-charts: helm repo add bitnami https://charts.bitnami.com/bitnami @@ -21,7 +23,7 @@ setup-charts: helm repo add jetstack https://charts.jetstack.io helm repo add dadrus https://dadrus.github.io/heimdall/charts helm repo add datawire https://app.getambassador.io - helm repo add haproxy-ingress https://haproxy-ingress.github.io/charts + helm repo add haproxy https://haproxy-ingress.github.io/charts helm repo update ## Installs Grafana @@ -96,8 +98,8 @@ install-nginx-ingress-controller: --wait install-contour-ingress-controller: - helm upgrade --install contour-ingress-controller bitnami/contour \ - -n contour-ingress-controller --create-namespace \ + helm upgrade --install contour-controller bitnami/contour \ + -n contour-controller --create-namespace \ --version {{contour_version}} \ -f contour/helm-values.yaml # used only to configure a global auth server @@ -107,35 +109,33 @@ install-emissary-ingress-controller: kubectl apply -f https://app.getambassador.io/yaml/emissary/${app_version}/emissary-crds.yaml kubectl wait --timeout=90s --for=condition=available deployment emissary-apiext -n emissary-system - helm upgrade --install emissary-ingress datawire/emissary-ingress \ - -n emissary-ingress-controller --create-namespace \ + helm upgrade --install emissary datawire/emissary \ + -n emissary-controller --create-namespace \ --version {{emissary_version}} - kubectl -n emissary-ingress-controller wait --for condition=available --timeout=90s deploy -lapp.kubernetes.io/instance=emissary-ingress + kubectl -n emissary-controller wait --for condition=available --timeout=90s deploy -lapp.kubernetes.io/instance=emissary - kubectl apply -f - < Date: Thu, 2 May 2024 13:15:24 +0200 Subject: [PATCH 2/4] initial and almost empty integration guide for envoy gateway --- .../content/guides/proxies/envoy_gateway.adoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/content/guides/proxies/envoy_gateway.adoc diff --git a/docs/content/guides/proxies/envoy_gateway.adoc b/docs/content/guides/proxies/envoy_gateway.adoc new file mode 100644 index 000000000..d21b9e359 --- /dev/null +++ b/docs/content/guides/proxies/envoy_gateway.adoc @@ -0,0 +1,19 @@ +--- +title: "Envoy Gateway Integration" +date: 2024-05-02T13:02:43+02:00 +draft: false +weight: 13 +menu: + guides: + parent: "API Gateways & Proxies" +description: This guide explains how to integrate heimdall with Envoy Gateway. +--- + +:toc: + +https://gateway.envoyproxy.io[Envoy Gateway] is an open source project for managing https://www.envoyproxy.io/[Envoy Proxy] as a Kubernetes-based application gateway by making use of the https://gateway-api.sigs.k8s.io/[Gateway API] resources. + +== Prerequisites + +* Integration with Envoy proxy requires heimdall being operated in link:{{< relref "/docs/concepts/operating_modes.adoc#_decision_mode" >}}[Decision Operation Mode]. + From c35df71030a04476e82ddc03cab084f8709334ae Mon Sep 17 00:00:00 2001 From: Dimitrij Drus Date: Fri, 3 May 2024 17:59:18 +0200 Subject: [PATCH 3/4] integration guide is almost done --- .../content/guides/proxies/envoy_gateway.adoc | 111 +++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/docs/content/guides/proxies/envoy_gateway.adoc b/docs/content/guides/proxies/envoy_gateway.adoc index d21b9e359..056aff112 100644 --- a/docs/content/guides/proxies/envoy_gateway.adoc +++ b/docs/content/guides/proxies/envoy_gateway.adoc @@ -15,5 +15,114 @@ https://gateway.envoyproxy.io[Envoy Gateway] is an open source project for manag == Prerequisites -* Integration with Envoy proxy requires heimdall being operated in link:{{< relref "/docs/concepts/operating_modes.adoc#_decision_mode" >}}[Decision Operation Mode]. +* A kubernetes cluster +* Deployed Envoy Gateway (See https://gateway.envoyproxy.io/v1.0.1/install/[here] for installation options) +* Deployed https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.GatewayClass[`GatewayClass`] resource that matches Envoy Gateway's configured `controllerName` (typically `gateway.envoyproxy.io/gatewayclass-controller`), as well as a deployed https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource. +* heimdall installed and operated in link:{{< relref "/docs/concepts/operating_modes.adoc#_decision_mode" >}}[Decision Operation Mode]. +== Integration Options + +Technically, the integration happens the same way as with link:{{< relref "/guides/proxies/envoy.adoc" >}}[Envoy] itself by making use of the https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_authz/v3/ext_authz.proto.html[External Authorization] filter, and can be done in two ways: + +* either via HTTP +* or via gRPC (recommended) + +In both cases, the filter calls an external gRPC or HTTP service (here heimdall) to check whether an incoming HTTP request is authorized or not. If heimdall responses with `2xx` the request is forwarded to the upstream service, otherwise the response from heimdall is returned to the caller. + +In case of Envoy Gateway the abovesaid configuration happens via a https://gateway.envoyproxy.io/contributions/design/security-policy/[`SecurityPolicy`] custom resource, that can be linked to a https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`], https://gateway-api.sigs.k8s.io/api-types/httproute[`HTTPRoute`], or a https://gateway-api.sigs.k8s.io/api-types/grpcroute[`GRPCRoute`] resource. + +== Global Configuration + +NOTE: As of today, there is a limitation in the implementation of the Envoy Gateway - it does not allow cross-namespace reference of external auth services (see also https://github.com/envoyproxy/gateway/issues/3322[envoyproxy/gateway#3322]). That means, both the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource and heimdall must be deployed in the same namespace. + +To integrate heimdall with the gateway globally, that is, each and every request will be forwarded to heimdall for authentication and authorization purposes first, create a https://gateway.envoyproxy.io/contributions/design/security-policy/[`SecurityPolicy`] as shown below in the namespace, the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource is deployed into. + +[source, yaml] +---- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: ext-auth-heimdall # <1> + namespace: heimdall # <2> +spec: + targetRef: # <3> + group: gateway.networking.k8s.io + kind: Gateway + name: eg + namespace: heimdall + extAuth: + grpc: + backendRef: # <4> + name: heimdall + port: 4456 + namespace: heimdall +---- +<1> The name of the `SecurityPolicy`. You can change it to any other value if you like. +<2> The namespace for the policy. It must be the same namespace the `Gateway` resource and heimdall are deployed into. So change it to your namespace. +<3> Defines the `Gateway` resource, this policy should be applied to. Change the `name` property to the name of your `Gateway` resource and the `namespace` property to the proper namespace (same as in 2) +<4> Defines the reference to the heimdall `Service` using the gRPC protocol. Change the `name` and the `namespace` to the proper values of your setup. + +== Route-level Configuration + +The integration on the route level happens similar to the link:{{< relref "#_global_configuration" >}}[global integration]. The difference is that the https://gateway.envoyproxy.io/contributions/design/security-policy/[`SecurityPolicy`] is applied to an https://gateway-api.sigs.k8s.io/api-types/httproute[`HTTPRoute`] as shown below and not the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource. + +[source, yaml] +---- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: ext-auth-example # <1> + namespace: myapp # <2> +spec: + targetRef: # <3> + group: gateway.networking.k8s.io + kind: HTTPRoute + name: myapp + extAuth: + grpc: + backendRef: # <4> + name: heimdall + port: 4456 + namespace: heimdall +---- +<1> The name of the `SecurityPolicy`. You can change it to any other value if you like. +<2> The namespace for the policy. It must be the same namespace the `HTTPRoute` resource is deployed into, so the namespace, your application is deployed to. So change it to your namespace. +<3> Defines the `HTTPRoute` resource, this policy should be applied to. Change the `name` property to the name of your `HTTPRoute` resource and the `namespace` property to the proper namespace (same as in 2) +<4> Defines the reference to the heimdall `Service` using the gRPC protocol. Change the `name` and the `namespace` to the proper values of your setup. + +== Security Considerations + +The configuration options shown above are highly insecure, as the communication from the gateway to heimdall happens over plain HTTP. Therefore, tt is highly recommended to enable TLS. This can be achieved by enabling TLS for heimdall and attaching a https://gateway-api.sigs.k8s.io/api-types/backendtlspolicy/[`BackendTLSPolicy`] resource shown below to heimdall's https://kubernetes.io/docs/concepts/services-networking/service/[`Service`]. + +[source, yaml] +---- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: BackendTLSPolicy +metadata: + name: heimdall-btls + namespace: heimdall # <1> +spec: + targetRef: # <2> + group: '' + kind: Service + namespace: heimdall + name: heimdall + sectionName: "4456" + tls: # <3> + caCertRefs: + - name: demo-ca # <4> + group: '' + kind: ConfigMap + hostname: heimdall # <5> +---- +<1> Change it to the namespace in which heimdall is deployed +<2> The reference to heimdall's `Service`. Change the `name` and the `namespace` to the proper values. +<3> Here we configure the reference to the `ConfigMap` with the certificate of the CA, used to issue a TLS server authentication certificate for heimdall, as well as the hostname used by heimdall (and present in the SAN extension of heimdall's TLS certificate). The `ConfigMap` must be in the same namespace as the `BackendTLSPolicy`. +<4> The name of the `ConfigMap`. Change it to the proper value. +<5> The expected hostname used by heimdall. Change it to the proper value. + +== Additional Resources + +* A fully working example with Envoy Gateway is also available on https://github.com/dadrus/heimdall/tree/main/examples[GitHub]. +* You can find the official external authentication guide for Envoy Gateway https://gateway.envoyproxy.io/v1.0.1/tasks/security/ext-auth/[here]. It contains a fully working setup with a demo application. +* https://gateway.envoyproxy.io/v1.0.1/tasks/security/secure-gateways/[Secure Gateways] is a highly recommended read as well. From bc08400840bf7db67f5458382a9ec1f6fd24162d Mon Sep 17 00:00:00 2001 From: Dimitrij Drus Date: Mon, 6 May 2024 14:17:36 +0200 Subject: [PATCH 4/4] guide for envoy gateway finalized --- .../content/guides/proxies/envoy_gateway.adoc | 10 ++--- examples/README.md | 2 +- examples/kubernetes/Justfile | 1 - examples/kubernetes/README.md | 2 +- .../quickstarts/heimdall/reference-grant.yaml | 44 ------------------- 5 files changed, 7 insertions(+), 52 deletions(-) delete mode 100644 examples/kubernetes/quickstarts/heimdall/reference-grant.yaml diff --git a/docs/content/guides/proxies/envoy_gateway.adoc b/docs/content/guides/proxies/envoy_gateway.adoc index 056aff112..f02181666 100644 --- a/docs/content/guides/proxies/envoy_gateway.adoc +++ b/docs/content/guides/proxies/envoy_gateway.adoc @@ -31,9 +31,9 @@ In both cases, the filter calls an external gRPC or HTTP service (here heimdall) In case of Envoy Gateway the abovesaid configuration happens via a https://gateway.envoyproxy.io/contributions/design/security-policy/[`SecurityPolicy`] custom resource, that can be linked to a https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`], https://gateway-api.sigs.k8s.io/api-types/httproute[`HTTPRoute`], or a https://gateway-api.sigs.k8s.io/api-types/grpcroute[`GRPCRoute`] resource. -== Global Configuration +NOTE: As of today, there is a limitation in the implementation of the Envoy Gateway - it does not allow cross-namespace reference of external auth services (see also https://github.com/envoyproxy/gateway/issues/3322[envoyproxy/gateway#3322]). That means, the https://gateway-api.sigs.k8s.io/api-types/httproute[`HTTPRoute`], the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource and heimdall must be deployed in the same namespace. -NOTE: As of today, there is a limitation in the implementation of the Envoy Gateway - it does not allow cross-namespace reference of external auth services (see also https://github.com/envoyproxy/gateway/issues/3322[envoyproxy/gateway#3322]). That means, both the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource and heimdall must be deployed in the same namespace. +== Global Configuration To integrate heimdall with the gateway globally, that is, each and every request will be forwarded to heimdall for authentication and authorization purposes first, create a https://gateway.envoyproxy.io/contributions/design/security-policy/[`SecurityPolicy`] as shown below in the namespace, the https://gateway-api.sigs.k8s.io/api-types/gateway[`Gateway`] resource is deployed into. @@ -72,12 +72,12 @@ apiVersion: gateway.envoyproxy.io/v1alpha1 kind: SecurityPolicy metadata: name: ext-auth-example # <1> - namespace: myapp # <2> + namespace: heimdall # <2> spec: targetRef: # <3> group: gateway.networking.k8s.io kind: HTTPRoute - name: myapp + name: heimdall extAuth: grpc: backendRef: # <4> @@ -92,7 +92,7 @@ spec: == Security Considerations -The configuration options shown above are highly insecure, as the communication from the gateway to heimdall happens over plain HTTP. Therefore, tt is highly recommended to enable TLS. This can be achieved by enabling TLS for heimdall and attaching a https://gateway-api.sigs.k8s.io/api-types/backendtlspolicy/[`BackendTLSPolicy`] resource shown below to heimdall's https://kubernetes.io/docs/concepts/services-networking/service/[`Service`]. +The configuration options shown above are highly insecure, as the communication from the gateway to heimdall happens over plain HTTP. Therefore, it is highly recommended to enable TLS. This can be achieved by enabling TLS for heimdall and attaching a https://gateway-api.sigs.k8s.io/api-types/backendtlspolicy/[`BackendTLSPolicy`] resource shown below to heimdall's https://kubernetes.io/docs/concepts/services-networking/service/[`Service`]. [source, yaml] ---- diff --git a/examples/README.md b/examples/README.md index fb84eb068..8179a68a8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,4 +8,4 @@ To be able to run the docker compose examples, you'll need Docker and docker-com To be able to run the Kubernetes based examples, you'll need just, kubectl, kustomize, helm and a k8s cluster. Latter can also be created locally using kind. The examples are indeed using it. -Please note: The main branch may have breaking changes (see pending release PRs for details under https://github.com/dadrus/heimdall/pulls) which would make the usage of the referenced heimdall images impossible (even though the configuration files and rules reflect the latest changes). In such situations you'll have to build a heimdall image by yourself and update the setups to use it. \ No newline at end of file +**Note:** The main branch may have breaking changes (see pending release PRs for details under https://github.com/dadrus/heimdall/pulls) which would make the usage of the referenced heimdall images impossible (even though the configuration files and rules reflect the latest changes). In such situations you'll have to use the dev image or build a heimdall image by yourself and update the setups to use it. \ No newline at end of file diff --git a/examples/kubernetes/Justfile b/examples/kubernetes/Justfile index 73e27d41f..b504f357a 100644 --- a/examples/kubernetes/Justfile +++ b/examples/kubernetes/Justfile @@ -198,7 +198,6 @@ install-heimdall router=default_router: kubectl apply -f quickstarts/heimdall/emissary-auth-service.yaml elif [ "{{router}}" == "envoygw" ]; then kubectl apply -f quickstarts/heimdall/backend-tls-policy.yaml - #kubectl apply -f quickstarts/heimdall/reference-grant.yaml kubectl apply -f quickstarts/heimdall/envoygw-security-policy.yaml fi diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index e30433a7e..38c649e09 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -2,7 +2,7 @@ This directory contains working examples described in the getting started, as well as in the integration guides of the documentation. The demonstration of the decision operation mode is done via integration with the corresponding ingress controllers. As of now, these are [Contour](https://projectcontour.io), the [NGINX Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/) and [HAProxy Ingress Controller](https://haproxy-ingress.github.io/). -**Note:** The main branch may have breaking changes (see pending release PRs for details under https://github.com/dadrus/heimdall/pulls) which would make the usage of the referenced heimdall images impossible (even though the configuration files and rules reflect the latest changes). In such situations you'll have to build a heimdall image by yourself and update the setups to use it. +**Note:** The main branch may have breaking changes (see pending release PRs for details under https://github.com/dadrus/heimdall/pulls) which would make the usage of the referenced heimdall images impossible (even though the configuration files and rules reflect the latest changes). In such situations you'll have to use the dev image or build a heimdall image by yourself and update the setups to use it. # Prerequisites diff --git a/examples/kubernetes/quickstarts/heimdall/reference-grant.yaml b/examples/kubernetes/quickstarts/heimdall/reference-grant.yaml deleted file mode 100644 index 48d82e148..000000000 --- a/examples/kubernetes/quickstarts/heimdall/reference-grant.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: ReferenceGrant -metadata: - name: ext-auth-heimdall-rg-hr - namespace: heimdall -spec: - from: - - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: default - to: - - group: "" - kind: Service - name: heimdall ---- -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: ReferenceGrant -metadata: - name: ext-auth-heimdall-rg-gr - namespace: heimdall -spec: - from: - - group: gateway.networking.k8s.io - kind: GRPCRoute - namespace: default - to: - - group: "" - kind: Service - name: heimdall ---- -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: ReferenceGrant -metadata: - name: ext-auth-heimdall-rg-gw - namespace: heimdall -spec: - from: - - group: gateway.networking.k8s.io - kind: Gateway - namespace: default - to: - - group: "" - kind: Service - name: heimdall \ No newline at end of file