feat(fallback): support custom entities (#6286)

* feat(fallback): support custom entities

* address review comments

* use existing plugin and custom entitiy

CHANGELOG.md

@@ -95,6 +95,11 @@ Adding a new version? You'll need three changes:
 
 ## Unreleased
 
+### Added
+
+- `KongCustomEntity` is now supported by the `FallbackConfiguration` feature.
+  [#6286](https://github.com/Kong/kubernetes-ingress-controller/pull/6286)
+
 ### Fixed
 
 - Services using `Secret`s containing the same certificate as client certificates

internal/dataplane/fallback/cache_to_graph_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	incubatorv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/incubator/v1alpha1"
 )
 
@@ -332,6 +333,49 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "cache with KongCustomEntities and its dependencies",
+			cache: cacheStoresFromObjs(t,
+				&kongv1alpha1.KongCustomEntity{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-entity-kong-plugin",
+						Namespace: testNamespace,
+					},
+					Spec: kongv1alpha1.KongCustomEntitySpec{
+						ParentRef: &kongv1alpha1.ObjectReference{
+							Kind:  lo.ToPtr("KongPlugin"),
+							Group: lo.ToPtr(kongv1alpha1.GroupVersion.Group),
+							Name:  "test-plugin",
+						},
+					},
+				},
+				&kongv1alpha1.KongCustomEntity{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-entity-kong-cluster-plugin",
+						Namespace: testNamespace,
+					},
+					Spec: kongv1alpha1.KongCustomEntitySpec{
+						ParentRef: &kongv1alpha1.ObjectReference{
+							Kind:  lo.ToPtr("KongClusterPlugin"),
+							Group: lo.ToPtr(kongv1alpha1.GroupVersion.Group),
+							Name:  "test-cluster-plugin",
+						},
+					},
+				},
+				testKongPlugin(t, "test-plugin"),
+				testKongClusterPlugin(t, "test-cluster-plugin"),
+			),
+			expectedAdjacencyMap: map[string][]string{
+				"configuration.konghq.com/KongPlugin:test-namespace/test-plugin": {
+					"configuration.konghq.com/KongCustomEntity:test-namespace/test-entity-kong-plugin",
+				},
+				"configuration.konghq.com/KongClusterPlugin:test-namespace/test-cluster-plugin": {
+					"configuration.konghq.com/KongCustomEntity:test-namespace/test-entity-kong-cluster-plugin",
+				},
+				"configuration.konghq.com/KongCustomEntity:test-namespace/test-entity-kong-plugin":         {},
+				"configuration.konghq.com/KongCustomEntity:test-namespace/test-entity-kong-cluster-plugin": {},
+			},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

internal/dataplane/fallback/graph_dependencies.go

@@ -52,6 +52,8 @@ func ResolveDependencies(cache store.CacheStores, obj client.Object) ([]client.O
 		return resolveTCPIngressDependencies(cache, obj), nil
 	case *incubatorv1alpha1.KongServiceFacade:
 		return resolveKongServiceFacadeDependencies(cache, obj), nil
+	case *kongv1alpha1.KongCustomEntity:
+		return resolveKongCustomEntityDependencies(cache, obj), nil
 	// Object types that have no dependencies.
 	case *netv1.IngressClass,
 		*corev1.Secret,
@@ -63,10 +65,6 @@ func ResolveDependencies(cache store.CacheStores, obj client.Object) ([]client.O
 		*kongv1alpha1.IngressClassParameters,
 		*kongv1alpha1.KongVault:
 		return nil, nil
-	case *kongv1alpha1.KongCustomEntity:
-		// TODO: KongCustomEnity is not supported in failure domain yet.
-		// https://github.com/Kong/kubernetes-ingress-controller/issues/6122
-		return nil, nil
 	default:
 		return nil, fmt.Errorf("unsupported object type: %T", obj)
 	}

internal/dataplane/fallback/graph_dependencies_kong.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 	incubatorv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/incubator/v1alpha1"
 )
@@ -140,3 +141,36 @@ func resolveTCPIngressDependencies(cache store.CacheStores, tcpIngress *kongv1be
 func resolveKongServiceFacadeDependencies(cache store.CacheStores, kongServiceFacade *incubatorv1alpha1.KongServiceFacade) []client.Object {
 	return resolveDependenciesForServiceLikeObj(cache, kongServiceFacade)
 }
+
+// resolveKongCustomEntityDependencies resolves potential dependencies for a KongCustomEntities object:
+// - KongPlugin
+// - KongClusterPlugin.
+func resolveKongCustomEntityDependencies(cache store.CacheStores, obj *kongv1alpha1.KongCustomEntity) []client.Object {
+	if obj.Spec.ParentRef == nil {
+		return nil
+	}
+
+	parentRef := *obj.Spec.ParentRef
+	groupMatches := parentRef.Group != nil && *parentRef.Group == kongv1.GroupVersion.Group
+
+	if isKongPlugin := parentRef.Kind != nil && *parentRef.Kind == "KongPlugin" && groupMatches; isKongPlugin {
+		// TODO: Cross-namespace references are not supported yet.
+		if parentRef.Namespace != nil && *parentRef.Namespace != "" &&
+			*parentRef.Namespace != obj.GetNamespace() {
+			return nil
+		}
+		if plugin, exists, err := cache.Plugin.GetByKey(
+			fmt.Sprintf("%s/%s", obj.GetNamespace(), parentRef.Name),
+		); err == nil && exists {
+			return []client.Object{plugin.(client.Object)}
+		}
+	}
+
+	if isKongClusterPlugin := parentRef.Kind != nil && *parentRef.Kind == "KongClusterPlugin" && groupMatches; isKongClusterPlugin {
+		if plugin, exists, err := cache.ClusterPlugin.GetByKey(parentRef.Name); err == nil && exists {
+			return []client.Object{plugin.(client.Object)}
+		}
+	}
+
+	return nil
+}

internal/dataplane/fallback/graph_dependencies_kong_test.go

@@ -3,12 +3,14 @@ package fallback_test
 import (
 	"testing"
 
+	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/annotations"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 	incubatorv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/incubator/v1alpha1"
 )
@@ -735,3 +737,103 @@ func TestResolveDependencies_TCPIngress(t *testing.T) {
 		runResolveDependenciesTest(t, tc)
 	}
 }
+
+func TestResolveDependencies_KongCustomEntity(t *testing.T) {
+	testCases := []resolveDependenciesTestCase{
+		{
+			name: "no dependencies - parent reference is nil",
+			object: &kongv1alpha1.KongCustomEntity{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-custom-entity",
+					Namespace: testNamespace,
+				},
+				Spec: kongv1alpha1.KongCustomEntitySpec{
+					EntityType: "test-entity",
+					ParentRef:  nil,
+				},
+			},
+			cache: cacheStoresFromObjs(t,
+				testKongPlugin(t, "1"),
+				testKongClusterPlugin(t, "1"),
+			),
+			expected: []client.Object{},
+		},
+		{
+			name: "parent reference to KongPlugin",
+			object: &kongv1alpha1.KongCustomEntity{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-custom-entity",
+					Namespace: testNamespace,
+				},
+				Spec: kongv1alpha1.KongCustomEntitySpec{
+					EntityType: "test-entity",
+					ParentRef: &kongv1alpha1.ObjectReference{
+						Name:  "1",
+						Kind:  lo.ToPtr("KongPlugin"),
+						Group: lo.ToPtr(kongv1alpha1.GroupVersion.Group),
+					},
+				},
+			},
+			cache: cacheStoresFromObjs(t,
+				testKongPlugin(t, "1"),
+				testKongClusterPlugin(t, "1"),
+			),
+			expected: []client.Object{
+				testKongPlugin(t, "1"),
+			},
+		},
+		{
+			name: "parent reference to KongClusterPlugin",
+			object: &kongv1alpha1.KongCustomEntity{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-custom-entity",
+					Namespace: testNamespace,
+				},
+				Spec: kongv1alpha1.KongCustomEntitySpec{
+					EntityType: "test-entity",
+					ParentRef: &kongv1alpha1.ObjectReference{
+						Name:  "1",
+						Kind:  lo.ToPtr("KongClusterPlugin"),
+						Group: lo.ToPtr(kongv1alpha1.GroupVersion.Group),
+					},
+				},
+			},
+			cache: cacheStoresFromObjs(t,
+				testKongPlugin(t, "1"),
+				testKongClusterPlugin(t, "1"),
+			),
+			expected: []client.Object{
+				testKongClusterPlugin(t, "1"),
+			},
+		},
+		{
+			name: "parent reference to KongPlugin in a different namespace",
+			object: &kongv1alpha1.KongCustomEntity{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-custom-entity",
+					Namespace: testNamespace,
+				},
+				Spec: kongv1alpha1.KongCustomEntitySpec{
+					EntityType: "test-entity",
+					ParentRef: &kongv1alpha1.ObjectReference{
+						Name:      "1",
+						Namespace: lo.ToPtr("other-namespace"),
+						Kind:      lo.ToPtr("KongPlugin"),
+						Group:     lo.ToPtr(kongv1alpha1.GroupVersion.Group),
+					},
+				},
+			},
+			cache: cacheStoresFromObjs(t,
+				testKongPlugin(t, "1", func(p *kongv1.KongPlugin) {
+					p.Namespace = "other-namespace"
+				}),
+				testKongClusterPlugin(t, "1"),
+			),
+			expected: []client.Object{},
+		},
+	}
+
+	for _, tc := range testCases {
+		runResolveDependenciesTest(t, tc)
+	}
+}

internal/dataplane/fallback/helpers_test.go

@@ -66,13 +66,17 @@ func testKongServiceFacade(t *testing.T, name string) *incubatorv1alpha1.KongSer
 	})
 }
 
-func testKongPlugin(t *testing.T, name string) *kongv1.KongPlugin {
-	return helpers.WithTypeMeta(t, &kongv1.KongPlugin{
+func testKongPlugin(t *testing.T, name string, modifiers ...func(p *kongv1.KongPlugin)) *kongv1.KongPlugin {
+	p := helpers.WithTypeMeta(t, &kongv1.KongPlugin{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: testNamespace,
 		},
 	})
+	for _, mod := range modifiers {
+		mod(p)
+	}
+	return p
 }
 
 func testKongClusterPlugin(t *testing.T, name string) *kongv1.KongClusterPlugin {

internal/dataplane/kong_client_golden_test.go

@@ -48,6 +48,7 @@ var (
 			// Feature flags that are directly propagated from the feature gates get their defaults.
 			FillIDs:           defaults.Enabled(featuregates.FillIDsFeature),
 			KongServiceFacade: defaults.Enabled(featuregates.KongServiceFacade),
+			KongCustomEntity:  defaults.Enabled(featuregates.KongCustomEntity),
 		}
 	}
 )
@@ -370,7 +371,33 @@ func extractObjectsFromYAML(t *testing.T, filePath string) [][]byte {
 type fakeSchemaServiceProvier struct{}
 
 func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService {
-	return translator.UnavailableSchemaService{}
+	return fakeSchemaService{}
+}
+
+// fakeSchemaService is a stub implementation of the kong.AbstractSchemaService interface returning hardcoded schemas
+// for testing purposes.
+type fakeSchemaService struct{}
+
+func (f fakeSchemaService) Get(_ context.Context, entityType string) (kong.Schema, error) {
+	switch entityType {
+	case "degraphql_routes":
+		return kong.Schema{
+			"fields": []interface{}{
+				map[string]interface{}{
+					"service": map[string]interface{}{
+						"type":      "foreign",
+						"reference": "services",
+					},
+				},
+			},
+		}, nil
+	default:
+		return kong.Schema{}, nil
+	}
+}
+
+func (f fakeSchemaService) Validate(context.Context, kong.EntityType, any) (bool, string, error) {
+	return true, "", nil
 }
 
 func buildPostConfigErrorResponseWithBrokenObjects(brokenObjects []client.Object) []byte {

internal/dataplane/testdata/golden/fallback-config-kong-custom-entities-ee/custom-entities-on_golden.yaml

@@ -0,0 +1,66 @@
+_format_version: "3.0"
+degraphql_routes:
+- query: query{ contacts { name } }
+  uri: /contacts
+- query: query{ contacts { name } }
+  uri: /contacts
+plugins:
+- config:
+    graphql_server_path: /v1/graphql
+  name: degraphql
+  route: default.i1.s1..80
+  tags:
+  - k8s-name:p1
+  - k8s-namespace:default
+  - k8s-kind:KongPlugin
+  - k8s-uid:e97ca177-6bd2-425f-b3b1-10a521ac02a0
+  - k8s-group:configuration.konghq.com
+  - k8s-version:v1
+services:
+- connect_timeout: 60000
+  host: s1.default.80.svc
+  id: 9e262d22-26f2-5bcb-8d57-c5229fb8c5a2
+  name: default.s1.80
+  path: /
+  port: 80
+  protocol: http
+  read_timeout: 60000
+  retries: 5
+  routes:
+  - https_redirect_status_code: 426
+    id: 226e9894-47fd-5dd2-9af0-4b49a2b45c5e
+    name: default.i1.s1..80
+    path_handling: v0
+    paths:
+    - /
+    preserve_host: true
+    protocols:
+    - http
+    - https
+    regex_priority: 0
+    request_buffering: true
+    response_buffering: true
+    strip_path: true
+    tags:
+    - k8s-name:i1
+    - k8s-namespace:default
+    - k8s-kind:Ingress
+    - k8s-uid:39ec2b6a-f919-4308-ad2f-07511ab424dd
+    - k8s-group:networking.k8s.io
+    - k8s-version:v1
+  tags:
+  - k8s-name:s1
+  - k8s-namespace:default
+  - k8s-kind:Service
+  - k8s-uid:ddc15c38-7f1b-41cb-8b57-46465a7f4427
+  - k8s-version:v1
+  write_timeout: 60000
+upstreams:
+- algorithm: round-robin
+  name: s1.default.80.svc
+  tags:
+  - k8s-name:s1
+  - k8s-namespace:default
+  - k8s-kind:Service
+  - k8s-uid:ddc15c38-7f1b-41cb-8b57-46465a7f4427
+  - k8s-version:v1

internal/dataplane/testdata/golden/fallback-config-kong-custom-entities-ee/custom-entities-on_settings.yaml

@@ -0,0 +1,2 @@
+feature_flags:
+  KongCustomEntity: true