support configuring gateway policy via annotation (#607)

Signed-off-by: spacewander <spacewanderlzx@gmail.com>

.github/workflows/lint.yml

@@ -57,6 +57,13 @@ jobs:
           echo "Generated files are not up-to-date. Please run 'make gen-crd-code' and commit changes."
           exit 1
         fi
+        pushd controller/
+        make manifests generate
+        if ! git diff --exit-code; then
+          echo "Generated files are not up-to-date. Please run 'make manifests generate' under controller/ and commit changes."
+          exit 1
+        fi
+        popd
         make gen-helm-docs
         if ! git diff --exit-code; then
           echo "Generated files are not up-to-date. Please run 'make gen-helm-docs' and commit changes."

controller/internal/controller/httpfilterpolicy_controller.go

@@ -423,7 +423,13 @@ func (r *HTTPFilterPolicyReconciler) resolveIstioGateway(ctx context.Context,
 		}
 	}
 
-	initState.AddPolicyForIstioGateway(policy, &gateway)
+	return r.resolveWithIstioGateway(ctx, &gateway, policy, initState)
+}
+
+func (r *HTTPFilterPolicyReconciler) resolveWithIstioGateway(_ context.Context,
+	gateway *istiov1a3.Gateway, policy *mosniov1.HTTPFilterPolicy, initState *translation.InitState) error {
+
+	initState.AddPolicyForIstioGateway(policy, gateway)
 	policy.SetAccepted(gwapiv1a2.PolicyReasonAccepted)
 	return nil
 }
@@ -602,6 +608,36 @@ func (r *HTTPFilterPolicyReconciler) policyToTranslationState(ctx context.Contex
 			return nil, fmt.Errorf("failed to list Istio Gateway: %w", err)
 		}
 
+		if config.EnableEmbeddedMode() {
+			for _, gw := range gateways.Items {
+				ann := gw.GetAnnotations()
+				if ann == nil || ann[constant.AnnotationHTTPFilterPolicy] == "" {
+					continue
+				}
+
+				var policy mosniov1.HTTPFilterPolicy
+				err := json.Unmarshal([]byte(ann[constant.AnnotationHTTPFilterPolicy]), &policy)
+				if err != nil {
+					log.Errorf("failed to unmarshal policy out from Gateway, err: %v, name: %s, namespace: %s", err, gw.Name, gw.Namespace)
+					continue
+				}
+				// We require the embedded policy to be valid, otherwise it's costly to validate and hard to report the error.
+
+				policy.Namespace = gw.Namespace
+				// Name convention is "embedded-$kind-$name"
+				policy.Name = "embedded-gateway-" + gw.Name
+				err = r.resolveWithIstioGateway(ctx, gw, &policy, initState)
+				if err != nil {
+					return nil, err
+				}
+
+				key := getK8sKey(gw.Namespace, gw.Name)
+				istioGwIdx[key] = append(istioGwIdx[key], &policy)
+			}
+
+			// istioGatewayIndexer will be updated at the end of this method
+		}
+
 		for _, gw := range gateways.Items {
 			initState.AddIstioGateway(gw)
 		}

controller/tests/integration/controller/httpfilterpolicy_controller_gateway_test.go

@@ -16,6 +16,7 @@ package controller
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"time"
 
@@ -27,6 +28,7 @@ import (
 	gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
 	"mosn.io/htnn/controller/internal/config"
+	"mosn.io/htnn/controller/internal/translation"
 	"mosn.io/htnn/controller/tests/pkg"
 	mosniov1 "mosn.io/htnn/types/apis/v1"
 )
@@ -265,6 +267,79 @@ var _ = Describe("HTTPFilterPolicy controller, for gateway", func() {
 			}, timeout, interval).Should(BeTrue())
 		})
 
+		It("deal with embedded HTTPFilterPolicy", func() {
+			ctx := context.Background()
+			input := []map[string]interface{}{}
+			mustReadHTTPFilterPolicy("istio_gateway_embedded_hfp", &input)
+
+			for _, in := range input {
+				obj := pkg.MapToObj(in)
+				Expect(k8sClient.Create(ctx, obj)).Should(Succeed())
+			}
+
+			var ef *istiov1a3.EnvoyFilter
+			Eventually(func() bool {
+				var envoyfilters istiov1a3.EnvoyFilterList
+				if err := k8sClient.List(ctx, &envoyfilters); err != nil {
+					return false
+				}
+				for _, e := range envoyfilters.Items {
+					if e.Name == "htnn-h-default" {
+						ef = e
+						return true
+					}
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+
+			Expect(len(ef.Spec.ConfigPatches)).To(Equal(2))
+			for _, cp := range ef.Spec.ConfigPatches {
+				if cp.ApplyTo != istioapi.EnvoyFilter_EXTENSION_CONFIG {
+					continue
+				}
+
+				b, _ := cp.Patch.Value.MarshalJSON()
+				Expect(string(b)).To(ContainSubstring(`"hostName":"peter"`))
+
+				var info translation.Info
+				ann := ef.Annotations["htnn.mosn.io/info"]
+				json.Unmarshal([]byte(ann), &info)
+				Expect(info.HTTPFilterPolicies).To(ConsistOf([]string{"default/embedded-gateway-default-embedded"}))
+			}
+		})
+
+		It("deal with embedded HTTPFilterPolicy, ignore invalid embedded HTTPFilterPolicy", func() {
+			ctx := context.Background()
+			input := []map[string]interface{}{}
+			mustReadHTTPFilterPolicy("istio_gateway_embedded_invalid_hfp", &input)
+
+			for _, in := range input {
+				obj := pkg.MapToObj(in)
+				Expect(k8sClient.Create(ctx, obj)).Should(Succeed())
+			}
+
+			var envoyfilters istiov1a3.EnvoyFilterList
+			Eventually(func() bool {
+				if err := k8sClient.List(ctx, &envoyfilters); err != nil {
+					return false
+				}
+				for _, ef := range envoyfilters.Items {
+					if ef.Name == "htnn-h-default" {
+						for _, cp := range ef.Spec.ConfigPatches {
+							v := cp.Patch.Value.AsMap()
+							name := v["name"].(string)
+							if name == "htnn-default-0.0.0.0_8889-golang-filter" && cp.ApplyTo == istioapi.EnvoyFilter_EXTENSION_CONFIG {
+								pv := v["typed_config"].(map[string]interface{})["plugin_config"].(map[string]interface{})["value"].(map[string]interface{})
+								if len(pv) == 0 {
+									return true
+								}
+							}
+						}
+					}
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+		})
 	})
 
 	Context("When generating LDS plugin configuration via ECDS (k8s Gateway)", func() {

controller/tests/integration/controller/httpfilterpolicy_controller_route_test.go

@@ -661,7 +661,7 @@ var _ = Describe("HTTPFilterPolicy controller, for route", func() {
 		It("deal with embedded HTTPFilterPolicy", func() {
 			ctx := context.Background()
 			input := []map[string]interface{}{}
-			mustReadHTTPFilterPolicy("virtualservice_embeded_hfp", &input)
+			mustReadHTTPFilterPolicy("virtualservice_embedded_hfp", &input)
 
 			var virtualService *istiov1a3.VirtualService
 			for _, in := range input {
@@ -780,7 +780,7 @@ var _ = Describe("HTTPFilterPolicy controller, for route", func() {
 		It("deal with embedded HTTPFilterPolicy, ignore invalid embedded HTTPFilterPolicy", func() {
 			ctx := context.Background()
 			input := []map[string]interface{}{}
-			mustReadHTTPFilterPolicy("virtualservice_embeded_invalid_hfp", &input)
+			mustReadHTTPFilterPolicy("virtualservice_embedded_invalid_hfp", &input)
 
 			for _, in := range input {
 				obj := pkg.MapToObj(in)
@@ -799,7 +799,7 @@ var _ = Describe("HTTPFilterPolicy controller, for route", func() {
 		It("deal with embedded HTTPFilterPolicy with SubPolicies", func() {
 			ctx := context.Background()
 			input := []map[string]interface{}{}
-			mustReadHTTPFilterPolicy("virtualservice_embeded_hfp_with_subpolicies", &input)
+			mustReadHTTPFilterPolicy("virtualservice_embedded_hfp_with_subpolicies", &input)
 
 			for _, in := range input {
 				obj := pkg.MapToObj(in)

controller/tests/integration/controller/serviceregistry_controller_test.go

@@ -285,10 +285,12 @@ var _ = Describe("ServiceRegistry controller", func() {
 				if err := k8sClient.List(ctx, &entries); err != nil {
 					return false
 				}
-				if len(entries.Items) != 1 {
-					return false
+				for _, e := range entries.Items {
+					if e.Name == "test.default-group.public.moon.nacos" {
+						return false
+					}
 				}
-				return entries.Items[0].Name == "test.default-group.public.earth.nacos"
+				return true
 			}, timeout, interval).Should(BeTrue())
 
 			Expect(k8sClient.Delete(ctx, registryEarth)).Should(Succeed())
@@ -354,7 +356,7 @@ var _ = Describe("ServiceRegistry controller", func() {
 					return false
 				}
 				// We use the number of ServiceEntries to indicate the reconciliation is done
-				return len(entries.Items) == 1
+				return len(entries.Items) > 0
 			}, timeout, interval).Should(BeTrue())
 			time.Sleep(1 * time.Second)
 			// Trigger a reconciliation

controller/tests/integration/controller/testdata/httpfilterpolicy/istio_gateway_embedded_hfp.yml

@@ -0,0 +1,36 @@
+- apiVersion: networking.istio.io/v1beta1
+  kind: Gateway
+  metadata:
+    name: default-embedded
+    namespace: default
+    annotations:
+      htnn.mosn.io/httpfilterpolicy: |
+        {"apiVersion":"htnn.mosn.io/v1","kind":"HTTPFilterPolicy","metadata":{"name":"policy","namespace":"default"},"spec":{"filters":{"demo":{"config":{"hostName":"peter"}}}}}
+  spec:
+    servers:
+    - hosts:
+      - default.local
+      port:
+        name: http
+        number: 8889
+        protocol: HTTP
+- apiVersion: networking.istio.io/v1beta1
+  kind: VirtualService
+  metadata:
+    name: vs
+    namespace: default
+  spec:
+    gateways:
+    - default-embedded
+    hosts:
+    - default.local
+    http:
+    - match:
+      - uri:
+          prefix: /
+      name: route
+      route:
+      - destination:
+          host: httpbin
+          port:
+            number: 8000

controller/tests/integration/controller/testdata/httpfilterpolicy/istio_gateway_embedded_invalid_hfp.yml

@@ -0,0 +1,17 @@
+- apiVersion: networking.istio.io/v1beta1
+  kind: Gateway
+  metadata:
+    name: gw-wrong-annotation
+    namespace: default
+    annotations:
+      # ignore annotation that can't be marshalled
+      htnn.mosn.io/httpfilterpolicy: |
+        []
+  spec:
+    servers:
+    - hosts:
+      - default.local
+      port:
+        name: http
+        number: 8889
+        protocol: HTTP

e2e/pkg/suite/suite.go

@@ -42,6 +42,7 @@ type Test struct {
 	Name      string
 	Manifests []string
 	Run       func(*testing.T, *Suite)
+	CleanUp   func(*testing.T, *Suite)
 }
 
 var (
@@ -92,7 +93,13 @@ func (suite *Suite) Run(t *testing.T) {
 					t.Fail()
 				}
 			}()
+
+			// We run CleanUp before test so we can keep the resources generated in the test
+			// when the test failed.
 			t.Logf("CleanUp test %q at %v", test.Name, time.Now())
+			if test.CleanUp != nil {
+				test.CleanUp(t, suite)
+			}
 			suite.cleanup(t)
 			for _, manifest := range test.Manifests {
 				k8s.Prepare(t, suite.Opt.Client, manifest)