feat(custom_entity) support cross-namespace reference for KongCustomEntity

CHANGELOG.md

@@ -102,6 +102,11 @@ Adding a new version? You'll need three changes:
 - It is now possible to disable synchronization of consumers to Konnect through the
   flag `--konnect-disable-consumers-sync`.
   [#6313](https://github.com/Kong/kubernetes-ingress-controller/pull/6313)
+- Allow `KongCustomEntity` to refer to plugins in another namespace via 
+  `spec.parentRef.namespace`. The reference is allowed only when there is a
+  `ReferenceGrant` in the namespace of the `KongPlugin` to grant permissions
+  to `KongCustomEntity` of referring to `KongPlugin`.
+  [#6289](https://github.com/Kong/kubernetes-ingress-controller/pull/6289)
 
 ### Fixed
 

internal/dataplane/kongstate/customentity.go

@@ -11,6 +11,7 @@ import (
 	"github.com/kong/go-kong/kong/custom"
 	"github.com/samber/lo"
 
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/annotations"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/failures"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
@@ -195,7 +196,11 @@ func (ks *KongState) FillCustomEntities(
 
 		// Fill the "foreign" fields if the entity has such fields referencing services/routes/consumers.
 		// First Find out possible foreign field combinations attached to the KCE resource.
-		foreignFieldCombinations := findCustomEntityForeignFields(logger, entity, schema, pluginRels, workspace)
+		foreignFieldCombinations, err := findCustomEntityForeignFields(logger, s, entity, schema, pluginRels, workspace)
+		if err != nil {
+			failuresCollector.PushResourceFailure(fmt.Sprintf("failed to find attached foreign entities for custom entity: %v", err), entity)
+			continue
+		}
 		// generate Kong entities from the fields in the KCE itself and attached foreign entities.
 		generatedEntities, err := generateCustomEntities(entity, foreignFieldCombinations)
 		if err != nil {
@@ -275,43 +280,55 @@ func (ks *KongState) sortCustomEntities() {
 	}
 }
 
-func findCustomEntityRelatedPlugin(k8sEntity *kongv1alpha1.KongCustomEntity) (string, bool) {
+func findCustomEntityRelatedPlugin(logger logr.Logger, cacheStore store.Storer, k8sEntity *kongv1alpha1.KongCustomEntity) (string, bool, error) {
 	// Find referred entity via the plugin in its spec.parentRef.
 	// Then we can fetch the referred service/route/consumer from the reference relations of the plugin.
 	parentRef := k8sEntity.Spec.ParentRef
-	var namespace string
 	// Abort if the parentRef is empty or does not refer to a plugin.
 	if parentRef == nil ||
 		(parentRef.Group == nil || *parentRef.Group != kongv1alpha1.GroupVersion.Group) {
-		return "", false
+		return "", false, nil
 	}
 	if parentRef.Kind == nil || (*parentRef.Kind != "KongPlugin" && *parentRef.Kind != "KongClusterPlugin") {
-		return "", false
+		return "", false, nil
 	}
+
 	// Extract the plugin key to get the plugin relations.
-	if parentRef.Namespace == nil || *parentRef.Namespace == "" {
-		namespace = k8sEntity.Namespace
-	} else {
-		namespace = *parentRef.Namespace
+	paretRefNamespace := lo.FromPtrOr(parentRef.Namespace, "")
+	// if the namespace in parentRef is not same as the namespace of KCE itself, check if the reference is allowed by ReferenceGrant.
+	if paretRefNamespace != "" && paretRefNamespace != k8sEntity.Namespace {
+		paretRefNamespace, err := extractReferredPluginNamespace(logger, cacheStore, k8sEntity, annotations.NamespacedKongPlugin{
+			Namespace: paretRefNamespace,
+			Name:      parentRef.Name,
+		})
+		if err != nil {
+			return "", false, err
+		}
+		return paretRefNamespace + ":" + parentRef.Name, true, nil
 	}
-	return namespace + ":" + parentRef.Name, true
+
+	return k8sEntity.Namespace + ":" + parentRef.Name, true, nil
 }
 
 func findCustomEntityForeignFields(
 	logger logr.Logger,
+	cacheStore store.Storer,
 	k8sEntity *kongv1alpha1.KongCustomEntity,
 	schema EntitySchema,
 	pluginRelEntities PluginRelatedEntitiesRefs,
 	workspace string,
-) [][]entityForeignFieldValue {
-	pluginKey, ok := findCustomEntityRelatedPlugin(k8sEntity)
+) ([][]entityForeignFieldValue, error) {
+	pluginKey, ok, err := findCustomEntityRelatedPlugin(logger, cacheStore, k8sEntity)
+	if err != nil {
+		return nil, err
+	}
 	if !ok {
-		return nil
+		return nil, nil
 	}
 	// Get the relations with other entities of the plugin.
 	rels, ok := pluginRelEntities.RelatedEntities[pluginKey]
 	if !ok {
-		return nil
+		return nil, nil
 	}
 
 	var (
@@ -378,7 +395,7 @@ func findCustomEntityForeignFields(
 		ret = append(ret, foreignFieldValues)
 	}
 
-	return ret
+	return ret, nil
 }
 
 // generateCustomEntities generates Kong entities from KongCustomEntity resource and combinations of attached foreign entities.

internal/dataplane/kongstate/customentity_test.go

@@ -18,12 +18,18 @@ import (
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/annotations"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/failures"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
 	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 )
 
+var customEntityTypeMeta = metav1.TypeMeta{
+	APIVersion: kongv1alpha1.GroupVersion.Group + "/" + kongv1alpha1.GroupVersion.Version,
+	Kind:       "KongCustomEntity",
+}
+
 func TestExtractEntityFieldDefinitions(t *testing.T) {
 	testCases := []struct {
 		name           string
@@ -224,6 +230,7 @@ func TestSortCustomEntities(t *testing.T) {
 
 func TestFindCustomEntityForeignFields(t *testing.T) {
 	testCustomEntity := &kongv1alpha1.KongCustomEntity{
+		TypeMeta: customEntityTypeMeta,
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "fake-entity",
@@ -267,9 +274,11 @@ func TestFindCustomEntityForeignFields(t *testing.T) {
 	}
 	testCases := []struct {
 		name                     string
+		referenceGrants          []*gatewayapi.ReferenceGrant
 		customEntity             *kongv1alpha1.KongCustomEntity
 		schema                   EntitySchema
 		pluginRelEntities        PluginRelatedEntitiesRefs
+		expectError              bool
 		foreignFieldCombinations [][]entityForeignFieldValue
 	}{
 		{
@@ -359,17 +368,167 @@ func TestFindCustomEntityForeignFields(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "attached to foreign plugin with reference grant allowed",
+			customEntity: func() *kongv1alpha1.KongCustomEntity {
+				e := testCustomEntity.DeepCopy()
+				e.Spec.ParentRef.Namespace = lo.ToPtr("another-namespace")
+				return e
+			}(),
+			schema: EntitySchema{
+				Fields: map[string]EntityField{
+					"foo":     {Name: "foo", Type: EntityFieldTypeString, Required: true},
+					"service": {Name: "service", Type: EntityFieldTypeForeign, Reference: "services"},
+				},
+			},
+			referenceGrants: []*gatewayapi.ReferenceGrant{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "another-namespace",
+						Name:      "grant-kce-to-plugin",
+					},
+					Spec: gatewayapi.ReferenceGrantSpec{
+						From: []gatewayapi.ReferenceGrantFrom{
+							{
+								Namespace: gatewayapi.Namespace("default"),
+								Group:     gatewayapi.Group(kongv1alpha1.GroupVersion.Group),
+								Kind:      "KongCustomEntity",
+							},
+						},
+						To: []gatewayapi.ReferenceGrantTo{
+							{
+								Group: gatewayapi.Group(kongv1alpha1.GroupVersion.Group),
+								Kind:  gatewayapi.Kind("KongPlugin"),
+							},
+						},
+					},
+				},
+			},
+			pluginRelEntities: PluginRelatedEntitiesRefs{
+				RelatedEntities: map[string]RelatedEntitiesRef{
+					"another-namespace:fake-plugin": {
+						Services: []*Service{
+							{
+								Service: kongService1,
+							},
+							{
+								Service: kongService2,
+							},
+						},
+					},
+				},
+			},
+			foreignFieldCombinations: [][]entityForeignFieldValue{
+				{
+					{fieldName: "service", foreignEntityType: kong.EntityTypeServices, foreignEntityID: "service1"},
+				},
+				{
+					{fieldName: "service", foreignEntityType: kong.EntityTypeServices, foreignEntityID: "service2"},
+				},
+			},
+		},
+		{
+			name: "attached to foreign plugin without reference grant allowed should fail",
+			customEntity: func() *kongv1alpha1.KongCustomEntity {
+				e := testCustomEntity.DeepCopy()
+				e.Spec.ParentRef.Namespace = lo.ToPtr("another-namespace")
+				return e
+			}(),
+			schema: EntitySchema{
+				Fields: map[string]EntityField{
+					"foo":     {Name: "foo", Type: EntityFieldTypeString, Required: true},
+					"service": {Name: "service", Type: EntityFieldTypeForeign, Reference: "services"},
+				},
+			},
+			pluginRelEntities: PluginRelatedEntitiesRefs{
+				RelatedEntities: map[string]RelatedEntitiesRef{
+					"another-namespace:fake-plugin": {
+						Services: []*Service{
+							{
+								Service: kongService1,
+							},
+							{
+								Service: kongService2,
+							},
+						},
+					},
+				},
+			},
+			expectError: true,
+		},
+		{
+			name: "attached to foreign plugin with misconfigured reference grant should fail",
+			customEntity: func() *kongv1alpha1.KongCustomEntity {
+				e := testCustomEntity.DeepCopy()
+				e.Spec.ParentRef.Namespace = lo.ToPtr("another-namespace")
+				return e
+			}(),
+			schema: EntitySchema{
+				Fields: map[string]EntityField{
+					"foo":     {Name: "foo", Type: EntityFieldTypeString, Required: true},
+					"service": {Name: "service", Type: EntityFieldTypeForeign, Reference: "services"},
+				},
+			},
+			referenceGrants: []*gatewayapi.ReferenceGrant{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default", // same namespace as KCE but not same as KongPlugin
+						Name:      "grant-kce-to-plugin-misconfigured",
+					},
+					Spec: gatewayapi.ReferenceGrantSpec{
+						From: []gatewayapi.ReferenceGrantFrom{
+							{
+								Namespace: gatewayapi.Namespace("default"),
+								Group:     gatewayapi.Group(kongv1alpha1.GroupVersion.Group),
+								Kind:      "KongCustomEntity",
+							},
+						},
+						To: []gatewayapi.ReferenceGrantTo{
+							{
+								Group: gatewayapi.Group(kongv1alpha1.GroupVersion.Group),
+								Kind:  gatewayapi.Kind("KongPlugin"),
+							},
+						},
+					},
+				},
+			},
+			pluginRelEntities: PluginRelatedEntitiesRefs{
+				RelatedEntities: map[string]RelatedEntitiesRef{
+					"another-namespace:fake-plugin": {
+						Services: []*Service{
+							{
+								Service: kongService1,
+							},
+							{
+								Service: kongService2,
+							},
+						},
+					},
+				},
+			},
+			expectError: true,
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			combinations := findCustomEntityForeignFields(
+			cacheStore, err := store.NewFakeStore(store.FakeObjects{
+				ReferenceGrants: tc.referenceGrants,
+			})
+			require.NoError(t, err)
+			combinations, err := findCustomEntityForeignFields(
 				logr.Discard(),
+				cacheStore,
 				tc.customEntity,
 				tc.schema,
 				tc.pluginRelEntities,
 				"",
 			)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
 			for index, combination := range combinations {
 				sort.SliceStable(combination, func(i, j int) bool {
 					return combination[i].fieldName < combination[j].fieldName
@@ -384,10 +543,6 @@ func TestFindCustomEntityForeignFields(t *testing.T) {
 }
 
 func TestKongState_FillCustomEntities(t *testing.T) {
-	customEntityTypeMeta := metav1.TypeMeta{
-		APIVersion: kongv1alpha1.GroupVersion.Group + "/" + kongv1alpha1.GroupVersion.Version,
-		Kind:       "KongCustomEntity",
-	}
 	kongService1 := kong.Service{
 		Name: kong.String("service1"),
 		ID:   kong.String("service1"),