diff --git a/pkg/apis/config/v1alpha1/interpretercontext_types.go b/pkg/apis/config/v1alpha1/interpretercontext_types.go index b268a19965be..5e674a758d4b 100644 --- a/pkg/apis/config/v1alpha1/interpretercontext_types.go +++ b/pkg/apis/config/v1alpha1/interpretercontext_types.go @@ -153,6 +153,13 @@ type DependentObjectReference struct { Namespace string `json:"namespace,omitempty"` // Name represents the name of the referent. - // +required - Name string `json:"name"` + // Name and LabelSelector cannot be empty at the same time. + // +optional + Name string `json:"name,omitempty"` + + // LabelSelector represents a label query over a set of resources. + // If name is not empty, labelSelector will be ignored. + // Name and LabelSelector cannot be empty at the same time. + // +optional + LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` } diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 14c1be8bb00f..e98a6005ca8e 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -7,6 +7,7 @@ package v1alpha1 import ( v1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -96,6 +97,11 @@ func (in *DependencyInterpretation) DeepCopy() *DependencyInterpretation { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DependentObjectReference) DeepCopyInto(out *DependentObjectReference) { *out = *in + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } return } @@ -368,7 +374,9 @@ func (in *ResourceInterpreterResponse) DeepCopyInto(out *ResourceInterpreterResp if in.Dependencies != nil { in, out := &in.Dependencies, &out.Dependencies *out = make([]DependentObjectReference, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.RawStatus != nil { in, out := &in.RawStatus, &out.RawStatus diff --git a/pkg/dependenciesdistributor/dependencies_distributor.go b/pkg/dependenciesdistributor/dependencies_distributor.go index fc15d9a5b401..7683a15fe8f0 100644 --- a/pkg/dependenciesdistributor/dependencies_distributor.go +++ b/pkg/dependenciesdistributor/dependencies_distributor.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" @@ -54,6 +53,13 @@ const ( bindingDependenciesAnnotationKey = "resourcebinding.karmada.io/dependencies" ) +// LabelsKey is the object key which is a unique identifier under a cluster, across all resources. +type LabelsKey struct { + keys.ClusterWideKey + // Labels is the labels of the referencing object. + Labels map[string]string +} + // DependenciesDistributor is to automatically propagate relevant resources. // Resource binding will be created when a resource(e.g. deployment) is matched by a propagation policy, we call it independent binding in DependenciesDistributor. // And when DependenciesDistributor works, it will create or update reference resource bindings of relevant resources(e.g. secret), which we call them attached bindings. @@ -111,7 +117,7 @@ func (d *DependenciesDistributor) OnUpdate(oldObj, newObj interface{}) { klog.V(4).Infof("Ignore update event of object (%s, kind=%s, %s) as specification no change", unstructuredOldObj.GetAPIVersion(), unstructuredOldObj.GetKind(), names.NamespacedKey(unstructuredOldObj.GetNamespace(), unstructuredOldObj.GetName())) return } - + d.OnAdd(oldObj) d.OnAdd(newObj) } @@ -123,7 +129,7 @@ func (d *DependenciesDistributor) OnDelete(obj interface{}) { // Reconcile performs a full reconciliation for the object referred to by the key. // The key will be re-queued if an error is non-nil. func (d *DependenciesDistributor) reconcile(key util.QueueKey) error { - clusterWideKey, ok := key.(keys.ClusterWideKey) + clusterWideKey, ok := key.(*LabelsKey) if !ok { klog.Error("Invalid key") return fmt.Errorf("invalid key") @@ -157,7 +163,7 @@ func (d *DependenciesDistributor) reconcile(key util.QueueKey) error { } // dependentObjectReferenceMatches tells if the given object is referred by current resource binding. -func dependentObjectReferenceMatches(objectKey keys.ClusterWideKey, referenceBinding *workv1alpha2.ResourceBinding) bool { +func dependentObjectReferenceMatches(objectKey *LabelsKey, referenceBinding *workv1alpha2.ResourceBinding) bool { dependencies, exist := referenceBinding.Annotations[bindingDependenciesAnnotationKey] if !exist { return false @@ -181,9 +187,17 @@ func dependentObjectReferenceMatches(objectKey keys.ClusterWideKey, referenceBin for _, dependence := range dependenciesSlice { if objectKey.GroupVersion().String() == dependence.APIVersion && objectKey.Kind == dependence.Kind && - objectKey.Namespace == dependence.Namespace && - objectKey.Name == dependence.Name { - return true + objectKey.Namespace == dependence.Namespace { + if len(dependence.Name) != 0 { + return dependence.Name == objectKey.Name + } + var selector labels.Selector + if selector, err = metav1.LabelSelectorAsSelector(dependence.LabelSelector); err != nil { + klog.Errorf("Failed to converts the LabelSelector of binding(%s/%s) dependencies(%s): %v", + referenceBinding.Namespace, referenceBinding.Name, dependencies, err) + return false + } + return selector.Matches(labels.Set(objectKey.Labels)) } } return false @@ -239,6 +253,60 @@ func (d *DependenciesDistributor) handleResourceBindingDeletion(namespace, name return d.removeScheduleResultFromAttachedBindings(namespace, name, attachedBindings) } +func (d *DependenciesDistributor) removeOrphanAttachedBindings(binding *workv1alpha2.ResourceBinding, dependencies []configv1alpha1.DependentObjectReference) error { + // remove orphan attached bindings + orphanBindings, err := d.findOrphanAttachedBindings(binding, dependencies) + if err != nil { + klog.Errorf("Failed to find orphan attached bindings for resourceBinding(%s/%s). Error: %v.", + binding.GetNamespace(), binding.GetName(), err) + return err + } + err = d.removeScheduleResultFromAttachedBindings(binding.Namespace, binding.Name, orphanBindings) + if err != nil { + klog.Errorf("Failed to remove orphan attached bindings by resourceBinding(%s/%s). Error: %v.", + binding.GetNamespace(), binding.GetName(), err) + return err + } + return nil +} + +func (d *DependenciesDistributor) handleDependentResource( + binding *workv1alpha2.ResourceBinding, + resource workv1alpha2.ObjectReference, + dependent configv1alpha1.DependentObjectReference) error { + switch { + case len(dependent.Name) != 0: + rawObject, err := helper.FetchResourceTemplate(d.DynamicClient, d.InformerManager, d.RESTMapper, resource) + if err != nil { + // do nothing if resource template not exist. + if apierrors.IsNotFound(err) { + return nil + } + return err + } + attachedBinding := buildAttachedBinding(binding, rawObject) + return d.createOrUpdateAttachedBinding(attachedBinding) + case dependent.LabelSelector != nil: + var selector labels.Selector + var err error + if selector, err = metav1.LabelSelectorAsSelector(dependent.LabelSelector); err != nil { + return err + } + rawObjects, err := helper.FetchResourceTemplateByLabelSelector(d.DynamicClient, d.InformerManager, d.RESTMapper, resource, selector) + if err != nil { + return err + } + for _, rawObject := range rawObjects { + attachedBinding := buildAttachedBinding(binding, rawObject) + if err := d.createOrUpdateAttachedBinding(attachedBinding); err != nil { + return err + } + } + return nil + } + return fmt.Errorf("the Name and LabelSelector in the DependentObjectReference of object (kind=%s %s/%s) cannot be empty at the same time", resource.Kind, resource.Namespace, resource.Name) +} + func (d *DependenciesDistributor) syncScheduleResultToAttachedBindings(binding *workv1alpha2.ResourceBinding, dependencies []configv1alpha1.DependentObjectReference) (err error) { defer func() { if err != nil { @@ -251,19 +319,7 @@ func (d *DependenciesDistributor) syncScheduleResultToAttachedBindings(binding * if err := d.recordDependencies(binding, dependencies); err != nil { return err } - - // remove orphan attached bindings - orphanBindings, err := d.findOrphanAttachedBindings(binding, dependencies) - if err != nil { - klog.Errorf("Failed to find orphan attached bindings for resourceBinding(%s/%s). Error: %v.", - binding.GetNamespace(), binding.GetName(), err) - return err - } - - err = d.removeScheduleResultFromAttachedBindings(binding.Namespace, binding.Name, orphanBindings) - if err != nil { - klog.Errorf("Failed to remove orphan attached bindings by resourceBinding(%s/%s). Error: %v.", - binding.GetNamespace(), binding.GetName(), err) + if err := d.removeOrphanAttachedBindings(binding, dependencies); err != nil { return err } @@ -286,20 +342,7 @@ func (d *DependenciesDistributor) syncScheduleResultToAttachedBindings(binding * d.InformerManager.ForResource(gvr, d.eventHandler) startInformerManager = true } - rawObject, err := helper.FetchResourceTemplate(d.DynamicClient, d.InformerManager, d.RESTMapper, resource) - if err != nil { - // do nothing if resource template not exist. - if apierrors.IsNotFound(err) { - continue - } - errs = append(errs, err) - continue - } - - attachedBinding := buildAttachedBinding(binding, rawObject) - if err := d.createOrUpdateAttachedBinding(attachedBinding); err != nil { - errs = append(errs, err) - } + errs = append(errs, d.handleDependentResource(binding, resource, dependent)) } if startInformerManager { d.InformerManager.Start() @@ -351,22 +394,72 @@ func (d *DependenciesDistributor) findOrphanAttachedBindings(independentBinding return nil, err } - dependenciesSets := sets.NewString() - for _, dependency := range dependencies { - key := generateDependencyKey(dependency.Kind, dependency.APIVersion, dependency.Namespace, dependency.Name) - dependenciesSets.Insert(key) + dependenciesMaps := make(map[string][]int, 0) + for index, dependency := range dependencies { + key := generateDependencyKey(dependency.Kind, dependency.APIVersion, dependency.Namespace) + dependenciesMaps[key] = append(dependenciesMaps[key], index) } var orphanAttachedBindings []*workv1alpha2.ResourceBinding for _, attachedBinding := range attachedBindings { - key := generateDependencyKey(attachedBinding.Spec.Resource.Kind, attachedBinding.Spec.Resource.APIVersion, attachedBinding.Spec.Resource.Namespace, attachedBinding.Spec.Resource.Name) - if !dependenciesSets.Has(key) { + key := generateDependencyKey(attachedBinding.Spec.Resource.Kind, attachedBinding.Spec.Resource.APIVersion, attachedBinding.Spec.Resource.Namespace) + dependencyIndexes, exist := dependenciesMaps[key] + if !exist { + orphanAttachedBindings = append(orphanAttachedBindings, attachedBinding) + continue + } + isOrphanAttachedBinding, err := d.isOrphanAttachedBindings(dependencies, dependencyIndexes, attachedBinding) + if err != nil { + return nil, err + } + if isOrphanAttachedBinding { orphanAttachedBindings = append(orphanAttachedBindings, attachedBinding) } } return orphanAttachedBindings, nil } +func (d *DependenciesDistributor) isOrphanAttachedBindings( + dependencies []configv1alpha1.DependentObjectReference, + dependencyIndexes []int, + attachedBinding *workv1alpha2.ResourceBinding) (bool, error) { + var resource = attachedBinding.Spec.Resource + for _, idx := range dependencyIndexes { + dependency := dependencies[idx] + switch { + case len(dependency.Name) != 0: + if dependency.Name == resource.Name { + return false, nil + } + case dependency.LabelSelector != nil: + var selector labels.Selector + var err error + if selector, err = metav1.LabelSelectorAsSelector(dependency.LabelSelector); err != nil { + return false, err + } + rawObject, err := helper.FetchResourceTemplate(d.DynamicClient, d.InformerManager, d.RESTMapper, workv1alpha2.ObjectReference{ + APIVersion: resource.APIVersion, + Kind: resource.Kind, + Namespace: resource.Namespace, + Name: resource.Name, + }) + if err != nil { + // do nothing if resource template not exist. + if apierrors.IsNotFound(err) { + continue + } + return false, err + } + if selector.Matches(labels.Set(rawObject.GetLabels())) { + return false, nil + } + default: + // can not reach here + } + } + return true, nil +} + func (d *DependenciesDistributor) listAttachedBindings(bindingNamespace, bindingName string) (res []*workv1alpha2.ResourceBinding, err error) { labelSet := generateBindingDependedLabels(bindingNamespace, bindingName) selector := labels.SelectorFromSet(labelSet) @@ -435,8 +528,21 @@ func (d *DependenciesDistributor) Start(ctx context.Context) error { klog.Infof("Starting dependencies distributor.") d.stopCh = ctx.Done() resourceWorkerOptions := util.Options{ - Name: "dependencies resource detector", - KeyFunc: func(obj interface{}) (util.QueueKey, error) { return keys.ClusterWideKeyFunc(obj) }, + Name: "dependencies resource detector", + KeyFunc: func(obj interface{}) (util.QueueKey, error) { + key, err := keys.ClusterWideKeyFunc(obj) + if err != nil { + return nil, err + } + metaInfo, err := meta.Accessor(obj) + if err != nil { // should not happen + return nil, fmt.Errorf("object has no meta: %v", err) + } + return &LabelsKey{ + ClusterWideKey: key, + Labels: metaInfo.GetLabels(), + }, nil + }, ReconcileFunc: d.reconcile, } d.eventHandler = fedinformer.NewHandlerOnEvents(d.OnAdd, d.OnUpdate, d.OnDelete) @@ -506,12 +612,12 @@ func generateBindingDependedLabelKey(bindingNamespace, bindingName string) strin return fmt.Sprintf(bindingDependedByLabelKeyPrefix + bindHashKey) } -func generateDependencyKey(kind, apiVersion, name, namespace string) string { +func generateDependencyKey(kind, apiVersion, namespace string) string { if len(namespace) == 0 { - return kind + "-" + apiVersion + "-" + name + return kind + "-" + apiVersion } - return kind + "-" + apiVersion + "-" + namespace + "-" + name + return kind + "-" + apiVersion + "-" + namespace } func buildAttachedBinding(binding *workv1alpha2.ResourceBinding, object *unstructured.Unstructured) *workv1alpha2.ResourceBinding { diff --git a/pkg/dependenciesdistributor/dependencies_distributor_test.go b/pkg/dependenciesdistributor/dependencies_distributor_test.go index 5d4147ecd9f2..aae9ccbd3044 100644 --- a/pkg/dependenciesdistributor/dependencies_distributor_test.go +++ b/pkg/dependenciesdistributor/dependencies_distributor_test.go @@ -1,17 +1,27 @@ package dependenciesdistributor import ( + "context" + "reflect" "testing" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" "github.com/karmada-io/karmada/pkg/util/fedinformer/keys" ) func Test_dependentObjectReferenceMatches(t *testing.T) { type args struct { - objectKey keys.ClusterWideKey + objectKey *LabelsKey referenceBinding *workv1alpha2.ResourceBinding } tests := []struct { @@ -22,12 +32,15 @@ func Test_dependentObjectReferenceMatches(t *testing.T) { { name: "test custom resource", args: args{ - objectKey: keys.ClusterWideKey{ - Group: "example-stgzr.karmada.io", - Version: "v1alpha1", - Kind: "Foot5zmh", - Namespace: "karmadatest-vpvll", - Name: "cr-fxzq6", + objectKey: &LabelsKey{ + ClusterWideKey: keys.ClusterWideKey{ + Group: "example-stgzr.karmada.io", + Version: "v1alpha1", + Kind: "Foot5zmh", + Namespace: "karmadatest-vpvll", + Name: "cr-fxzq6", + }, + Labels: nil, }, referenceBinding: &workv1alpha2.ResourceBinding{ ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ @@ -40,12 +53,15 @@ func Test_dependentObjectReferenceMatches(t *testing.T) { { name: "test configmap", args: args{ - objectKey: keys.ClusterWideKey{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - Namespace: "karmadatest-h46wh", - Name: "configmap-8w426", + objectKey: &LabelsKey{ + ClusterWideKey: keys.ClusterWideKey{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + Namespace: "karmadatest-h46wh", + Name: "configmap-8w426", + }, + Labels: nil, }, referenceBinding: &workv1alpha2.ResourceBinding{ ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ @@ -55,6 +71,28 @@ func Test_dependentObjectReferenceMatches(t *testing.T) { }, want: true, }, + { + name: "test labels", + args: args{ + objectKey: &LabelsKey{ + ClusterWideKey: keys.ClusterWideKey{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + Namespace: "test", + }, + Labels: map[string]string{ + "app": "test", + }, + }, + referenceBinding: &workv1alpha2.ResourceBinding{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + bindingDependenciesAnnotationKey: "[{\"apiVersion\":\"v1\",\"kind\":\"ConfigMap\",\"namespace\":\"test\",\"labelSelector\":{\"matchExpressions\":[{\"key\":\"app\",\"operator\":\"In\",\"values\":[\"test\"]}]}}]", + }}, + }, + }, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -65,3 +103,141 @@ func Test_dependentObjectReferenceMatches(t *testing.T) { }) } } + +func TestDependenciesDistributor_findOrphanAttachedBindingsByDependencies(t *testing.T) { + type fields struct { + DynamicClient dynamic.Interface + InformerManager genericmanager.SingleClusterInformerManager + RESTMapper meta.RESTMapper + } + type args struct { + dependencies []configv1alpha1.DependentObjectReference + dependencyIndexes []int + attachedBinding *workv1alpha2.ResourceBinding + } + tests := []struct { + name string + fields fields + args args + want bool + wantErr bool + }{ + { + name: "can't match labels", + fields: fields{ + DynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme), + InformerManager: func() genericmanager.SingleClusterInformerManager { + c := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "default", Labels: map[string]string{"bar": "bar"}}}) + m := genericmanager.NewSingleClusterInformerManager(c, 0, context.TODO().Done()) + m.Lister(corev1.SchemeGroupVersion.WithResource("pods")) + m.Start() + m.WaitForCacheSync() + return m + }(), + RESTMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + return m + }(), + }, + args: args{ + dependencies: []configv1alpha1.DependentObjectReference{ + { + APIVersion: "v1", + Kind: "Secret", + Namespace: "default", + Name: "test", + }, + { + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "bar": "foo", + }}, + }, + }, + dependencyIndexes: []int{1}, + attachedBinding: &workv1alpha2.ResourceBinding{ + Spec: workv1alpha2.ResourceBindingSpec{ + Resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + }, + }, + }, + want: true, + wantErr: false, + }, + { + name: "can't match name", + fields: fields{ + DynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme), + InformerManager: func() genericmanager.SingleClusterInformerManager { + c := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "default", Labels: map[string]string{"bar": "foo"}}}) + m := genericmanager.NewSingleClusterInformerManager(c, 0, context.TODO().Done()) + m.Lister(corev1.SchemeGroupVersion.WithResource("pods")) + m.Start() + m.WaitForCacheSync() + return m + }(), + RESTMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + return m + }(), + }, + args: args{ + dependencies: []configv1alpha1.DependentObjectReference{ + { + APIVersion: "v1", + Kind: "Secret", + Namespace: "default", + Name: "test", + }, + { + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + }, + dependencyIndexes: []int{1}, + attachedBinding: &workv1alpha2.ResourceBinding{ + Spec: workv1alpha2.ResourceBindingSpec{ + Resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + Name: "test2", + }, + }, + }, + }, + want: true, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &DependenciesDistributor{ + DynamicClient: tt.fields.DynamicClient, + InformerManager: tt.fields.InformerManager, + RESTMapper: tt.fields.RESTMapper, + } + got, err := d.isOrphanAttachedBindings(tt.args.dependencies, tt.args.dependencyIndexes, tt.args.attachedBinding) + if (err != nil) != tt.wantErr { + t.Errorf("findOrphanAttachedBindingsByDependencies() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("findOrphanAttachedBindingsByDependencies() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 67b691024701..c947a89c0151 100755 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1811,16 +1811,23 @@ func schema_pkg_apis_config_v1alpha1_DependentObjectReference(ref common.Referen }, "name": { SchemaProps: spec.SchemaProps{ - Description: "Name represents the name of the referent.", - Default: "", + Description: "Name represents the name of the referent. Name and LabelSelector cannot be empty at the same time.", Type: []string{"string"}, Format: "", }, }, + "labelSelector": { + SchemaProps: spec.SchemaProps{ + Description: "LabelSelector represents a label query over a set of resources. If name is not empty, labelSelector will be ignored. Name and LabelSelector cannot be empty at the same time.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), + }, + }, }, - Required: []string{"apiVersion", "kind", "name"}, + Required: []string{"apiVersion", "kind"}, }, }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, } } diff --git a/pkg/util/helper/binding.go b/pkg/util/helper/binding.go index 7b896b34ff55..1cf6885063af 100644 --- a/pkg/util/helper/binding.go +++ b/pkg/util/helper/binding.go @@ -251,6 +251,56 @@ func FetchResourceTemplate( return unstructuredObj, nil } +// FetchResourceTemplateByLabelSelector fetches the resource template by label selector to be propagated. +// Any updates to this resource template are not recommended as it may come from the informer cache. +// We should abide by the principle of making a deep copy first and then modifying it. +// See issue: https://github.com/karmada-io/karmada/issues/3878. +func FetchResourceTemplateByLabelSelector( + dynamicClient dynamic.Interface, + informerManager genericmanager.SingleClusterInformerManager, + restMapper meta.RESTMapper, + resource workv1alpha2.ObjectReference, + selector labels.Selector, +) ([]*unstructured.Unstructured, error) { + gvr, err := restmapper.GetGroupVersionResource(restMapper, schema.FromAPIVersionAndKind(resource.APIVersion, resource.Kind)) + if err != nil { + klog.Errorf("Failed to get GVR from GVK(%s/%s), Error: %v", resource.APIVersion, resource.Kind, err) + return nil, err + } + var objectList []runtime.Object + if len(resource.Namespace) == 0 { + // cluster-scoped resource + objectList, err = informerManager.Lister(gvr).List(selector) + } else { + objectList, err = informerManager.Lister(gvr).ByNamespace(resource.Namespace).List(selector) + } + var objects []*unstructured.Unstructured + if err != nil || len(objectList) == 0 { + // fall back to call api server in case the cache has not been synchronized yet + klog.Warningf("Failed to get resource template (%s/%s/%s) from cache, Error: %v. Fall back to call api server.", + resource.Kind, resource.Namespace, resource.Name, err) + unstructuredList, err := dynamicClient.Resource(gvr).Namespace(resource.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + klog.Errorf("Failed to get resource template (%s/%s/%s) from api server, Error: %v", + resource.Kind, resource.Namespace, resource.Name, err) + return nil, err + } + for i := range unstructuredList.Items { + objects = append(objects, &unstructuredList.Items[i]) + } + } + + for i := range objectList { + unstructuredObj, err := ToUnstructured(objectList[i]) + if err != nil { + klog.Errorf("Failed to transform object(%s/%s), Error: %v", resource.Namespace, resource.Name, err) + return nil, err + } + objects = append(objects, unstructuredObj) + } + return objects, nil +} + // GetClusterResourceBindings returns a ClusterResourceBindingList by labels. func GetClusterResourceBindings(c client.Client, ls labels.Set) (*workv1alpha2.ClusterResourceBindingList, error) { bindings := &workv1alpha2.ClusterResourceBindingList{} diff --git a/pkg/util/helper/binding_test.go b/pkg/util/helper/binding_test.go index 4b98cd2a83af..35ddc2b10fe1 100644 --- a/pkg/util/helper/binding_test.go +++ b/pkg/util/helper/binding_test.go @@ -801,6 +801,154 @@ func TestFetchWorkload(t *testing.T) { } } +func TestFetchWorkloadByLabelSelector(t *testing.T) { + type args struct { + dynamicClient dynamic.Interface + informerManager func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager + restMapper meta.RESTMapper + resource workv1alpha2.ObjectReference + selector *metav1.LabelSelector + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + { + name: "kind is not registered", + args: args{ + dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme), + informerManager: func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager { + return genericmanager.NewSingleClusterInformerManager(dynamicfake.NewSimpleDynamicClient(scheme.Scheme), 0, stopCh) + }, + restMapper: meta.NewDefaultRESTMapper(nil), + resource: workv1alpha2.ObjectReference{APIVersion: "v1", Kind: "Pod"}, + }, + want: 0, + wantErr: true, + }, + { + name: "namespace scope: get from client", + args: args{ + dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "default", Labels: map[string]string{"foo": "foo"}}}), + informerManager: func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager { + return genericmanager.NewSingleClusterInformerManager(dynamicfake.NewSimpleDynamicClient(scheme.Scheme), 0, stopCh) + }, + restMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + return m + }(), + resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + }, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "foo"}}, + }, + want: 1, + wantErr: false, + }, + { + name: "namespace scope: get from cache", + args: args{ + dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme), + informerManager: func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager { + c := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "default", Labels: map[string]string{"bar": "foo"}}}) + m := genericmanager.NewSingleClusterInformerManager(c, 0, stopCh) + m.Lister(corev1.SchemeGroupVersion.WithResource("pods")) + m.Start() + m.WaitForCacheSync() + return m + }, + restMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + return m + }(), + resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"bar": "foo"}}, + }, + want: 1, + wantErr: false, + }, + { + name: "cluster scope: get from client", + args: args{ + dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Labels: map[string]string{"bar": "bar"}}}), + informerManager: func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager { + return genericmanager.NewSingleClusterInformerManager(dynamicfake.NewSimpleDynamicClient(scheme.Scheme), 0, stopCh) + }, + restMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Node"), meta.RESTScopeRoot) + return m + }(), + resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Node", + Name: "node", + }, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"bar": "bar"}}, + }, + want: 1, + wantErr: false, + }, + { + name: "cluster scope: get from cache", + args: args{ + dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme.Scheme), + informerManager: func(stopCh <-chan struct{}) genericmanager.SingleClusterInformerManager { + c := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", Labels: map[string]string{"bar": "foo"}}}) + m := genericmanager.NewSingleClusterInformerManager(c, 0, stopCh) + m.Lister(corev1.SchemeGroupVersion.WithResource("nodes")) + m.Start() + m.WaitForCacheSync() + return m + }, + restMapper: func() meta.RESTMapper { + m := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + m.Add(corev1.SchemeGroupVersion.WithKind("Node"), meta.RESTScopeRoot) + return m + }(), + resource: workv1alpha2.ObjectReference{ + APIVersion: "v1", + Kind: "Node", + Name: "node", + }, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"bar": "foo"}}, + }, + want: 1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stopCh := make(chan struct{}) + mgr := tt.args.informerManager(stopCh) + selector, _ := metav1.LabelSelectorAsSelector(tt.args.selector) + got, err := FetchResourceTemplateByLabelSelector(tt.args.dynamicClient, mgr, tt.args.restMapper, tt.args.resource, selector) + if (err != nil) != tt.wantErr { + t.Errorf("FetchResourceTemplate() error = %v, wantErr %v", err, tt.wantErr) + return + } + if len(got) != tt.want { + t.Errorf("FetchResourceTemplate() got = %v, want %v", got, tt.want) + } + }) + } +} + func TestDeleteWorkByRBNamespaceAndName(t *testing.T) { type args struct { c client.Client