Skip to content

Commit

Permalink
feat: add resoruce id label
Browse files Browse the repository at this point in the history
Signed-off-by: jwcesign <jwcesign@gmail.com>
  • Loading branch information
jwcesign committed Nov 1, 2023
1 parent 60af4e8 commit 9c00889
Show file tree
Hide file tree
Showing 10 changed files with 124 additions and 34 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.6.0
github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
github.com/kr/pretty v0.3.1
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
github.com/olekukonko/tablewriter v0.0.5
Expand Down Expand Up @@ -100,7 +101,6 @@ require (
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/policy/v1alpha1/well_known_constants.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
package v1alpha1

const (
// PropagationPolicyIDLabel is the id of PropagationPolicy object.
PropagationPolicyIDLabel = "propagationpolicy.karmada.io/id"

// ClusterPropagationPolicyIDLabel is the id of ClusterPropagationPolicy object.
ClusterPropagationPolicyIDLabel = "clusterpropagationpolicy.karmada.io/id"

// PropagationPolicyUIDLabel is the uid of PropagationPolicy object.
PropagationPolicyUIDLabel = "propagationpolicy.karmada.io/uid"

Expand Down
9 changes: 9 additions & 0 deletions pkg/apis/work/v1alpha2/well_known_constants.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
package v1alpha2

const (
// ResourceBindingIDLabel is the ID of ResourceBinding object.
ResourceBindingIDLabel = "resourcebinding.karmada.io/id"

// ClusterResourceBindingIDLabel is the ID of ClusterResourceBinding object.
ClusterResourceBindingIDLabel = "clusterresourcebinding.karmada.io/id"

// WorkIDLabel is the ID of Work object.
WorkIDLabel = "work.karmada.io/id"

// ResourceBindingUIDLabel is the UID of ResourceBinding object.
ResourceBindingUIDLabel = "resourcebinding.karmada.io/uid"

Expand Down
16 changes: 12 additions & 4 deletions pkg/controllers/binding/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,15 +143,23 @@ func mergeLabel(workload *unstructured.Unstructured, workNamespace string, bindi
util.MergeLabel(workload, workv1alpha1.WorkNameLabel, names.GenerateWorkName(workload.GetKind(), workload.GetName(), workload.GetNamespace()))
util.MergeLabel(workload, util.ManagedByKarmadaLabel, util.ManagedByKarmadaLabelValue)
if scope == apiextensionsv1.NamespaceScoped {
util.RemoveLabels(workload, workv1alpha2.ResourceBindingUIDLabel)

bindingID := util.GetLabelValue(binding.GetLabels(), workv1alpha2.ResourceBindingIDLabel)
util.MergeLabel(workload, workv1alpha2.ResourceBindingReferenceKey, names.GenerateBindingReferenceKey(binding.GetNamespace(), binding.GetName()))
util.MergeLabel(workload, workv1alpha2.ResourceBindingUIDLabel, string(binding.GetUID()))
util.MergeLabel(workload, workv1alpha2.ResourceBindingIDLabel, bindingID)

workLabel[workv1alpha2.ResourceBindingReferenceKey] = names.GenerateBindingReferenceKey(binding.GetNamespace(), binding.GetName())
workLabel[workv1alpha2.ResourceBindingUIDLabel] = string(binding.GetUID())
workLabel[workv1alpha2.ResourceBindingIDLabel] = bindingID
} else {
util.RemoveLabels(workload, workv1alpha2.ClusterResourceBindingUIDLabel)

bindingID := util.GetLabelValue(binding.GetLabels(), workv1alpha2.ClusterResourceBindingIDLabel)
util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingReferenceKey, names.GenerateBindingReferenceKey("", binding.GetName()))
util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingUIDLabel, string(binding.GetUID()))
util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingIDLabel, bindingID)

workLabel[workv1alpha2.ClusterResourceBindingReferenceKey] = names.GenerateBindingReferenceKey("", binding.GetName())
workLabel[workv1alpha2.ClusterResourceBindingUIDLabel] = string(binding.GetUID())
workLabel[workv1alpha2.ClusterResourceBindingIDLabel] = bindingID
}
return workLabel
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/execution/execution_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,12 +190,12 @@ func (c *Controller) syncToClusters(clusterName string, work *workv1alpha1.Work)
for _, manifest := range work.Spec.Workload.Manifests {
workload := &unstructured.Unstructured{}
err := workload.UnmarshalJSON(manifest.Raw)
util.MergeLabel(workload, workv1alpha2.WorkUIDLabel, string(work.UID))
if err != nil {
klog.Errorf("Failed to unmarshal workload, error is: %v", err)
errs = append(errs, err)
continue
}
util.MergeLabel(workload, workv1alpha2.WorkIDLabel, util.GetLabelValue(work.Labels, workv1alpha2.WorkIDLabel))

if err = c.tryCreateOrUpdateWorkload(clusterName, workload); err != nil {
klog.Errorf("Failed to create or update resource(%v/%v) in the given member cluster %s, err is %v", workload.GetNamespace(), workload.GetName(), clusterName, err)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (r *HPAReplicasSyncer) Generic(e event.GenericEvent) bool {
}

func hasBeenPropagated(hpa *autoscalingv2.HorizontalPodAutoscaler) bool {
_, ppExist := hpa.GetLabels()[policyv1alpha1.PropagationPolicyUIDLabel]
_, cppExist := hpa.GetLabels()[policyv1alpha1.ClusterPropagationPolicyUIDLabel]
_, ppExist := hpa.GetLabels()[policyv1alpha1.PropagationPolicyIDLabel]
_, cppExist := hpa.GetLabels()[policyv1alpha1.ClusterPropagationPolicyIDLabel]
return ppExist || cppExist
}
5 changes: 5 additions & 0 deletions pkg/dependenciesdistributor/dependencies_distributor.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ import (
)

const (
// bindingDependedIdLabelKey is the resoruce id of the independent binding which the attached binding depends on.
bindingDependedIdLabelKey = "resourcebinding.karmada.io/depended-id"

// bindingDependedByLabelKeyPrefix is the prefix to a label key specifying an attached binding referred by which independent binding.
// the key is in the label of an attached binding which should be unique, because resource like secret can be referred by multiple deployments.
bindingDependedByLabelKeyPrefix = "resourcebinding.karmada.io/depended-by-"
Expand Down Expand Up @@ -630,6 +633,8 @@ func buildAttachedBinding(binding *workv1alpha2.ResourceBinding, object *unstruc
Clusters: binding.Spec.Clusters,
})

policyID := util.GetLabelValue(binding.Labels, workv1alpha2.ResourceBindingIDLabel)
dependedLabels = util.DedupeAndMergeLabels(dependedLabels, map[string]string{bindingDependedIdLabelKey: policyID})
return &workv1alpha2.ResourceBinding{
ObjectMeta: metav1.ObjectMeta{
Name: names.GenerateBindingName(object.GetKind(), object.GetName()),
Expand Down
101 changes: 78 additions & 23 deletions pkg/detector/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"sync"
"time"

"github.com/google/uuid"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -388,15 +389,16 @@ func (d *ResourceDetector) ApplyPolicy(object *unstructured.Unstructured, object
}
}()

if err := d.ClaimPolicyForObject(object, policy.Namespace, policy.Name, string(policy.UID)); err != nil {
policyID, err := d.ClaimPolicyForObject(object, policy)
if err != nil {
klog.Errorf("Failed to claim policy(%s) for object: %s", policy.Name, object)
return err
}

policyLabels := map[string]string{
policyv1alpha1.PropagationPolicyNamespaceLabel: policy.GetNamespace(),
policyv1alpha1.PropagationPolicyNameLabel: policy.GetName(),
policyv1alpha1.PropagationPolicyUIDLabel: string(policy.UID),
policyv1alpha1.PropagationPolicyIDLabel: policyID,
}
policyAnnotations := map[string]string{
policyv1alpha1.PropagationPolicyNamespaceAnnotation: policy.GetNamespace(),
Expand All @@ -417,6 +419,13 @@ func (d *ResourceDetector) ApplyPolicy(object *unstructured.Unstructured, object
return fmt.Errorf("failed to update binding due to different owner reference UID, will " +
"try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090")
}

// Delete following one line in release-1.9
delete(bindingCopy.Labels, workv1alpha2.ResourceBindingUIDLabel)
if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ResourceBindingIDLabel) == "" {
bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels,
map[string]string{workv1alpha2.ResourceBindingIDLabel: uuid.New().String()})
}
// Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists.
bindingCopy.Annotations = util.DedupeAndMergeAnnotations(bindingCopy.Annotations, binding.Annotations)
bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, binding.Labels)
Expand Down Expand Up @@ -469,14 +478,15 @@ func (d *ResourceDetector) ApplyClusterPolicy(object *unstructured.Unstructured,
}
}()

if err := d.ClaimClusterPolicyForObject(object, policy.Name, string(policy.UID)); err != nil {
policyID, err := d.ClaimClusterPolicyForObject(object, policy)
if err != nil {
klog.Errorf("Failed to claim cluster policy(%s) for object: %s", policy.Name, object)
return err
}

policyLabels := map[string]string{
policyv1alpha1.ClusterPropagationPolicyLabel: policy.GetName(),
policyv1alpha1.ClusterPropagationPolicyUIDLabel: string(policy.UID),
policyv1alpha1.ClusterPropagationPolicyLabel: policy.GetName(),
policyv1alpha1.ClusterPropagationPolicyIDLabel: policyID,
}
policyAnnotations := map[string]string{
policyv1alpha1.ClusterPropagationPolicyAnnotation: policy.GetName(),
Expand All @@ -500,6 +510,13 @@ func (d *ResourceDetector) ApplyClusterPolicy(object *unstructured.Unstructured,
return fmt.Errorf("failed to update binding due to different owner reference UID, will " +
"try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090")
}

// Delete following one line in release-1.9
delete(bindingCopy.Labels, workv1alpha2.ResourceBindingUIDLabel)
if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ResourceBindingIDLabel) == "" {
bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels,
map[string]string{workv1alpha2.ResourceBindingIDLabel: uuid.New().String()})
}
// Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists.
bindingCopy.Annotations = util.DedupeAndMergeAnnotations(bindingCopy.Annotations, binding.Annotations)
bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, binding.Labels)
Expand Down Expand Up @@ -614,51 +631,89 @@ func (d *ResourceDetector) GetUnstructuredObject(objectKey keys.ClusterWideKey)
return unstructuredObj, nil
}

func (d *ResourceDetector) getPropagationPolicyID(policy *policyv1alpha1.PropagationPolicy) (string, error) {
id := util.GetLabelValue(policy.GetLabels(), policyv1alpha1.PropagationPolicyIDLabel)
if id == "" {
id = uuid.New().String()
policy.Labels = util.DedupeAndMergeLabels(policy.Labels, map[string]string{policyv1alpha1.PropagationPolicyIDLabel: id})
if err := d.Client.Update(context.TODO(), policy); err != nil {
return id, err
}
}

return id, nil
}

// ClaimPolicyForObject set policy identifier which the object associated with.
func (d *ResourceDetector) ClaimPolicyForObject(object *unstructured.Unstructured, policyNamespace, policyName, policyUID string) error {
func (d *ResourceDetector) ClaimPolicyForObject(object *unstructured.Unstructured, policy *policyv1alpha1.PropagationPolicy) (string, error) {
policyID, err := d.getPropagationPolicyID(policy)
if err != nil {
klog.Errorf("Get PropagationPolicy(%s/%s) ID error:%v", policy.Namespace, policy.Name, err)
return "", err
}

objLabels := object.GetLabels()
if objLabels == nil {
objLabels = make(map[string]string)
} else if len(objLabels) > 0 {
// object has been claimed, don't need to claim again
if !excludeClusterPolicy(objLabels) &&
objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] == policyNamespace &&
objLabels[policyv1alpha1.PropagationPolicyNameLabel] == policyName {
return nil
objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] == policy.Namespace &&
objLabels[policyv1alpha1.PropagationPolicyNameLabel] == policy.Name {
return policyID, nil
}
}

objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] = policyNamespace
objLabels[policyv1alpha1.PropagationPolicyNameLabel] = policyName
objLabels[policyv1alpha1.PropagationPolicyUIDLabel] = policyUID
objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] = policy.Namespace
objLabels[policyv1alpha1.PropagationPolicyNameLabel] = policy.Name
objLabels[policyv1alpha1.PropagationPolicyIDLabel] = policyID

objectAnnotations := object.GetAnnotations()
if objectAnnotations == nil {
objectAnnotations = make(map[string]string)
}
objectAnnotations[policyv1alpha1.PropagationPolicyNamespaceAnnotation] = policyNamespace
objectAnnotations[policyv1alpha1.PropagationPolicyNameAnnotation] = policyName
objectAnnotations[policyv1alpha1.PropagationPolicyNamespaceAnnotation] = policy.Namespace
objectAnnotations[policyv1alpha1.PropagationPolicyNameAnnotation] = policy.Name

objectCopy := object.DeepCopy()
objectCopy.SetLabels(objLabels)
objectCopy.SetAnnotations(objectAnnotations)
return d.Client.Update(context.TODO(), objectCopy)
return policyID, d.Client.Update(context.TODO(), objectCopy)
}

func (d *ResourceDetector) getClusterPropagationPolicyID(policy *policyv1alpha1.ClusterPropagationPolicy) (string, error) {
id := util.GetLabelValue(policy.GetLabels(), policyv1alpha1.ClusterPropagationPolicyIDLabel)
if id == "" {
id = uuid.New().String()
policy.Labels = util.DedupeAndMergeLabels(policy.Labels, map[string]string{policyv1alpha1.ClusterPropagationPolicyIDLabel: id})
if err := d.Client.Update(context.TODO(), policy); err != nil {
return "", err
}
}

return id, nil
}

// ClaimClusterPolicyForObject set cluster identifier which the object associated with.
func (d *ResourceDetector) ClaimClusterPolicyForObject(object *unstructured.Unstructured, policyName, policyUID string) error {
// ClaimClusterPolicyForObject set cluster identifier which the object associated with
func (d *ResourceDetector) ClaimClusterPolicyForObject(object *unstructured.Unstructured, policy *policyv1alpha1.ClusterPropagationPolicy) (string, error) {
policyID, err := d.getClusterPropagationPolicyID(policy)
if err != nil {
klog.Errorf("Get ClusterPropagationPolicy(%s) ID error:%v", policy.Name, err)
return "", err
}

claimedName := util.GetLabelValue(object.GetLabels(), policyv1alpha1.ClusterPropagationPolicyLabel)
// object has been claimed, don't need to claim again
if claimedName == policyName {
return nil
if claimedName == policy.Name {
return policyID, nil
}

objectCopy := object.DeepCopy()
util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyLabel, policyName)
util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyUIDLabel, policyUID)
util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyLabel, policy.Name)
util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyIDLabel, policyID)

util.MergeAnnotation(objectCopy, policyv1alpha1.ClusterPropagationPolicyAnnotation, policyName)
return d.Client.Update(context.TODO(), objectCopy)
util.MergeAnnotation(objectCopy, policyv1alpha1.ClusterPropagationPolicyAnnotation, policy.Name)
return policyID, d.Client.Update(context.TODO(), objectCopy)
}

// BuildResourceBinding builds a desired ResourceBinding for object.
Expand Down
6 changes: 3 additions & 3 deletions pkg/detector/preemption.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func (d *ResourceDetector) preemptPropagationPolicy(resourceTemplate *unstructur
"Propagation policy(%s/%s) preempted propagation policy(%s/%s) successfully", policy.Namespace, policy.Name, claimedPolicyNamespace, claimedPolicyName)
}()

if err = d.ClaimPolicyForObject(resourceTemplate, policy.Namespace, policy.Name, string(policy.UID)); err != nil {
if _, err = d.ClaimPolicyForObject(resourceTemplate, policy); err != nil {
klog.Errorf("Failed to claim new propagation policy(%s/%s) on resource template(%s, kind=%s, %s): %v.", policy.Namespace, policy.Name,
resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err)
return err
Expand Down Expand Up @@ -147,7 +147,7 @@ func (d *ResourceDetector) preemptClusterPropagationPolicyDirectly(resourceTempl
"Propagation policy(%s/%s) preempted cluster propagation policy(%s) successfully", policy.Namespace, policy.Name, claimedPolicyName)
}()

if err = d.ClaimPolicyForObject(resourceTemplate, policy.Namespace, policy.Name, string(policy.UID)); err != nil {
if _, err = d.ClaimPolicyForObject(resourceTemplate, policy); err != nil {
klog.Errorf("Failed to claim new propagation policy(%s/%s) on resource template(%s, kind=%s, %s) directly: %v.", policy.Namespace, policy.Name,
resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err)
return err
Expand Down Expand Up @@ -197,7 +197,7 @@ func (d *ResourceDetector) preemptClusterPropagationPolicy(resourceTemplate *uns
"Cluster propagation policy(%s) preempted cluster propagation policy(%s) successfully", policy.Name, claimedPolicyName)
}()

if err = d.ClaimClusterPolicyForObject(resourceTemplate, policy.Name, string(policy.UID)); err != nil {
if _, err = d.ClaimClusterPolicyForObject(resourceTemplate, policy); err != nil {
klog.Errorf("Failed to claim new cluster propagation policy(%s) on resource template(%s, kind=%s, %s): %v.", policy.Name,
resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err)
return err
Expand Down
7 changes: 7 additions & 0 deletions pkg/util/helper/work.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/google/uuid"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -58,6 +59,12 @@ func CreateOrUpdateWork(client client.Client, workMeta metav1.ObjectMeta, resour
if !runtimeObject.DeletionTimestamp.IsZero() {
return fmt.Errorf("work %s/%s is being deleted", runtimeObject.GetNamespace(), runtimeObject.GetName())
}

// Delete following one line in release-1.9
delete(runtimeObject.Labels, workv1alpha2.WorkUIDLabel)
if util.GetLabelValue(runtimeObject.Labels, workv1alpha2.WorkIDLabel) == "" {
work.Labels = util.DedupeAndMergeLabels(work.Labels, map[string]string{workv1alpha2.WorkIDLabel: uuid.New().String()})
}
runtimeObject.Spec = work.Spec
runtimeObject.Labels = work.Labels
runtimeObject.Annotations = work.Annotations
Expand Down

0 comments on commit 9c00889

Please sign in to comment.