SSA: Implement request caching

config/manager/manager.yaml

@@ -38,6 +38,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.uid
+            - name: CAPI_DISABLE_SSA_REQUEST_TRACKER
+              value: ${CAPI_DISABLE_SSA_REQUEST_TRACKER:=""}
           ports:
             - containerPort: 9440
               name: healthz

go.mod

@@ -153,3 +153,5 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 )
+
+replace sigs.k8s.io/controller-runtime => github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5

go.sum

@@ -482,6 +482,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5 h1:uvjw527nGIFq6coUUbtx3dOQVmy3uoX60mJCnK85Vlk=
+github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -1025,8 +1027,6 @@ k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.14.2 h1:P6IwDhbsRWsBClt/8/h8Zy36bCuGuW5Op7MHpFrN/60=
-sigs.k8s.io/controller-runtime v0.14.2/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=

hack/observability/prometheus/values.yaml

@@ -14,6 +14,8 @@ pushgateway:
 # Scrape metrics from deployed providers
 extraScrapeConfigs: |
   - job_name: 'capi-providers'
+    # 15s is a bit often for production but helps to get metrics quicker for development.
+    scrape_interval: 15s
     metrics_path: /metrics
     kubernetes_sd_configs:
       - role: pod

hack/tools/go.mod

@@ -157,3 +157,5 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
+
+replace sigs.k8s.io/controller-runtime => github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5

hack/tools/go.sum

@@ -488,6 +488,8 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5 h1:uvjw527nGIFq6coUUbtx3dOQVmy3uoX60mJCnK85Vlk=
+github.com/sbueringer/controller-runtime v0.2.0-beta.1.0.20230301021438-864519a0e3e5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -1015,8 +1017,6 @@ oras.land/oras-go v1.2.2/go.mod h1:Apa81sKoZPpP7CDciE006tSZ0x3Q3+dOoBcMZ/aNxvw=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.14.2 h1:P6IwDhbsRWsBClt/8/h8Zy36bCuGuW5Op7MHpFrN/60=
-sigs.k8s.io/controller-runtime v0.14.2/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 sigs.k8s.io/controller-tools v0.11.3 h1:T1xzLkog9saiyQSLz1XOImu4OcbdXWytc5cmYsBeBiE=
 sigs.k8s.io/controller-tools v0.11.3/go.mod h1:qcfX7jfcfYD/b7lAhvqAyTbt/px4GpvN88WKLFFv7p8=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=

internal/controllers/machine/machine_controller.go

@@ -46,6 +46,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/external"
 	"sigs.k8s.io/cluster-api/controllers/noderefutil"
 	"sigs.k8s.io/cluster-api/controllers/remote"
+	"sigs.k8s.io/cluster-api/internal/util/ssa"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/collections"
@@ -58,6 +59,10 @@ import (
 const (
 	// controllerName defines the controller used when creating clients.
 	controllerName = "machine-controller"
+
+	// machineManagerName is the manager name used for Server-Side-Apply (SSA) operations
+	// in the Machine controller.
+	machineManagerName = "capi-machine"
 )
 
 var (
@@ -91,6 +96,7 @@ type Reconciler struct {
 	// nodeDeletionRetryTimeout determines how long the controller will retry deleting a node
 	// during a single reconciliation.
 	nodeDeletionRetryTimeout time.Duration
+	ssaRequestTracker        ssa.RequestTracker
 }
 
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -134,6 +140,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	r.externalTracker = external.ObjectTracker{
 		Controller: controller,
 	}
+	r.ssaRequestTracker = ssa.NewRequestTracker(mgr.GetScheme())
 	return nil
 }
 

internal/controllers/machine/machine_controller_noderef.go

@@ -33,6 +33,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/api/v1beta1/index"
 	"sigs.k8s.io/cluster-api/controllers/noderefutil"
+	"sigs.k8s.io/cluster-api/internal/util/ssa"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -124,13 +125,15 @@ func (r *Reconciler) reconcileNode(ctx context.Context, cluster *clusterv1.Clust
 		}
 	}
 
-	options := []client.PatchOption{
-		client.FieldOwner("capi-machine"),
-		client.ForceOwnership,
+	// We have to convert node to unstructured as ssa.Run expects original and updated to be of the same type.
+	originalNode := &unstructured.Unstructured{}
+	if err := remoteClient.Scheme().Convert(node, originalNode, nil); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to apply labels to the node: failed to convert node to Unstructured")
 	}
-	nodePatch := unstructuredNode(node.Name, node.UID, getManagedLabels(machine.Labels))
-	if err := remoteClient.Patch(ctx, nodePatch, client.Apply, options...); err != nil {
-		return ctrl.Result{}, errors.Wrap(err, "failed to apply patch label to the node")
+	updatedNode := unstructuredNode(node.Name, node.UID, getManagedLabels(machine.Labels))
+	_, err = ssa.Run(ctx, remoteClient, r.ssaRequestTracker, originalNode, updatedNode, machineManagerName)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to apply labels to the node")
 	}
 
 	// Do the remaining node health checks, then set the node health to true if all checks pass.

internal/controllers/machinedeployment/machinedeployment_controller.go

@@ -51,8 +51,6 @@ var (
 	machineDeploymentKind = clusterv1.GroupVersion.WithKind("MachineDeployment")
 )
 
-// machineDeploymentManagerName is the manager name used for Server-Side-Apply (SSA) operations
-// in the MachineDeployment controller.
 const machineDeploymentManagerName = "capi-machinedeployment"
 
 // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch
@@ -69,7 +67,8 @@ type Reconciler struct {
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 
-	recorder record.EventRecorder
+	recorder          record.EventRecorder
+	ssaRequestTracker ssa.RequestTracker
 }
 
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -106,6 +105,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	}
 
 	r.recorder = mgr.GetEventRecorderFor("machinedeployment-controller")
+	r.ssaRequestTracker = ssa.NewRequestTracker(mgr.GetScheme())
 	return nil
 }
 

internal/controllers/machinedeployment/machinedeployment_sync.go

@@ -40,6 +40,8 @@ import (
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil"
+	"sigs.k8s.io/cluster-api/internal/util/hash"
+	"sigs.k8s.io/cluster-api/internal/util/ssa"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
@@ -153,13 +155,10 @@ func (r *Reconciler) updateMachineSet(ctx context.Context, deployment *clusterv1
 	}
 
 	// Update the MachineSet to propagate in-place mutable fields from the MachineDeployment.
-	patchOptions := []client.PatchOption{
-		client.ForceOwnership,
-		client.FieldOwner(machineDeploymentManagerName),
-	}
-	if err := r.Client.Patch(ctx, updatedMS, client.Apply, patchOptions...); err != nil {
+	updatedMS, err = ssa.Run(ctx, r.Client, r.ssaRequestTracker, ms, updatedMS, machineDeploymentManagerName)
+	if err != nil {
 		r.recorder.Eventf(deployment, corev1.EventTypeWarning, "FailedUpdate", "Failed to update MachineSet %s: %v", klog.KObj(updatedMS), err)
-		return nil, errors.Wrapf(err, "failed to update MachineSet %s", klog.KObj(updatedMS))
+		return nil, err
 	}
 
 	log.V(4).Info("Updated MachineSet", "MachineSet", klog.KObj(updatedMS))
@@ -237,7 +236,7 @@ func (r *Reconciler) computeDesiredMachineSet(deployment *clusterv1.MachineDeplo
 		// As a result, we use the hash of the machine template while ignoring all in-place mutable fields, i.e. the
 		// machine template with only fields that could trigger a rollout for the machine-template-hash, making it
 		// independent of the changes to any in-place mutable fields.
-		templateHash, err := mdutil.ComputeSpewHash(mdutil.MachineTemplateDeepCopyRolloutFields(&deployment.Spec.Template))
+		templateHash, err := hash.Compute(mdutil.MachineTemplateDeepCopyRolloutFields(&deployment.Spec.Template))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to compute desired MachineSet: failed to compute machine template hash")
 		}

internal/controllers/machinedeployment/mdutil/util.go

@@ -19,13 +19,10 @@ package mdutil
 
 import (
 	"fmt"
-	"hash"
-	"hash/fnv"
 	"sort"
 	"strconv"
 	"strings"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -688,33 +685,6 @@ func CloneSelectorAndAddLabel(selector *metav1.LabelSelector, labelKey, labelVal
 	return newSelector
 }
 
-// SpewHashObject writes specified object to hash using the spew library
-// which follows pointers and prints actual values of the nested objects
-// ensuring the hash does not change when a pointer changes.
-func SpewHashObject(hasher hash.Hash, objectToWrite interface{}) error {
-	hasher.Reset()
-	printer := spew.ConfigState{
-		Indent:         " ",
-		SortKeys:       true,
-		DisableMethods: true,
-		SpewKeys:       true,
-	}
-
-	if _, err := printer.Fprintf(hasher, "%#v", objectToWrite); err != nil {
-		return fmt.Errorf("failed to write object to hasher")
-	}
-	return nil
-}
-
-// ComputeSpewHash computes the hash of a MachineTemplateSpec using the spew library.
-func ComputeSpewHash(objectToWrite interface{}) (uint32, error) {
-	machineTemplateSpecHasher := fnv.New32a()
-	if err := SpewHashObject(machineTemplateSpecHasher, objectToWrite); err != nil {
-		return 0, err
-	}
-	return machineTemplateSpecHasher.Sum32(), nil
-}
-
 // GetDeletingMachineCount gets the number of machines that are in the process of being deleted
 // in a machineList.
 func GetDeletingMachineCount(machineList *clusterv1.MachineList) int32 {

internal/controllers/machineset/machineset_controller.go

@@ -85,7 +85,8 @@ type Reconciler struct {
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 
-	recorder record.EventRecorder
+	ssaRequestTracker ssa.RequestTracker
+	recorder          record.EventRecorder
 }
 
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -122,6 +123,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	}
 
 	r.recorder = mgr.GetEventRecorderFor("machineset-controller")
+	r.ssaRequestTracker = ssa.NewRequestTracker(mgr.GetScheme())
 	return nil
 }
 
@@ -393,13 +395,10 @@ func (r *Reconciler) syncMachines(ctx context.Context, machineSet *clusterv1.Mac
 
 		// Update Machine to propagate in-place mutable fields from the MachineSet.
 		updatedMachine := r.computeDesiredMachine(machineSet, m)
-		patchOptions := []client.PatchOption{
-			client.ForceOwnership,
-			client.FieldOwner(machineSetManagerName),
-		}
-		if err := r.Client.Patch(ctx, updatedMachine, client.Apply, patchOptions...); err != nil {
+		updatedMachine, err := ssa.Run(ctx, r.Client, r.ssaRequestTracker, m, updatedMachine, machineSetManagerName)
+		if err != nil {
 			log.Error(err, "failed to update Machine", "Machine", klog.KObj(updatedMachine))
-			return errors.Wrapf(err, "failed to update Machine %q", klog.KObj(updatedMachine))
+			return err
 		}
 		machines[i] = updatedMachine
 
@@ -664,7 +663,7 @@ func (r *Reconciler) computeDesiredMachine(machineSet *clusterv1.MachineSet, exi
 
 // updateExternalObject updates the external object passed in with the
 // updated labels and annotations from the MachineSet.
-func (r *Reconciler) updateExternalObject(ctx context.Context, obj client.Object, machineSet *clusterv1.MachineSet) error {
+func (r *Reconciler) updateExternalObject(ctx context.Context, obj *unstructured.Unstructured, machineSet *clusterv1.MachineSet) error {
 	updatedObject := &unstructured.Unstructured{}
 	updatedObject.SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind())
 	updatedObject.SetNamespace(obj.GetNamespace())
@@ -676,14 +675,8 @@ func (r *Reconciler) updateExternalObject(ctx context.Context, obj client.Object
 	updatedObject.SetLabels(machineLabelsFromMachineSet(machineSet))
 	updatedObject.SetAnnotations(machineAnnotationsFromMachineSet(machineSet))
 
-	patchOptions := []client.PatchOption{
-		client.ForceOwnership,
-		client.FieldOwner(machineSetManagerName),
-	}
-	if err := r.Client.Patch(ctx, updatedObject, client.Apply, patchOptions...); err != nil {
-		return errors.Wrapf(err, "failed to update %s", klog.KObj(obj))
-	}
-	return nil
+	_, err := ssa.Run(ctx, r.Client, r.ssaRequestTracker, obj, updatedObject, machineSetManagerName)
+	return err
 }
 
 // machineLabelsFromMachineSet computes the labels the Machine created from this MachineSet should have.

internal/controllers/topology/cluster/cluster_controller.go

@@ -47,6 +47,7 @@ import (
 	"sigs.k8s.io/cluster-api/internal/hooks"
 	tlog "sigs.k8s.io/cluster-api/internal/log"
 	runtimeclient "sigs.k8s.io/cluster-api/internal/runtime/client"
+	"sigs.k8s.io/cluster-api/internal/util/ssa"
 	"sigs.k8s.io/cluster-api/internal/webhooks"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
@@ -85,6 +86,7 @@ type Reconciler struct {
 	patchEngine patches.Engine
 
 	patchHelperFactory structuredmerge.PatchHelperFactoryFunc
+	ssaRequestTracker  ssa.RequestTracker
 }
 
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -117,8 +119,9 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	}
 	r.patchEngine = patches.NewEngine(r.RuntimeClient)
 	r.recorder = mgr.GetEventRecorderFor("topology/cluster")
+	r.ssaRequestTracker = ssa.NewRequestTracker(mgr.GetScheme())
 	if r.patchHelperFactory == nil {
-		r.patchHelperFactory = serverSideApplyPatchHelperFactory(r.Client)
+		r.patchHelperFactory = serverSideApplyPatchHelperFactory(r.Client, r.ssaRequestTracker)
 	}
 	return nil
 }
@@ -394,9 +397,9 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, cluster *clusterv1.Clu
 }
 
 // serverSideApplyPatchHelperFactory makes use of managed fields provided by server side apply and is used by the controller.
-func serverSideApplyPatchHelperFactory(c client.Client) structuredmerge.PatchHelperFactoryFunc {
+func serverSideApplyPatchHelperFactory(c client.Client, ssaRequestTracker ssa.RequestTracker) structuredmerge.PatchHelperFactoryFunc {
 	return func(ctx context.Context, original, modified client.Object, opts ...structuredmerge.HelperOption) (structuredmerge.PatchHelper, error) {
-		return structuredmerge.NewServerSidePatchHelper(ctx, original, modified, c, opts...)
+		return structuredmerge.NewServerSidePatchHelper(ctx, original, modified, c, ssaRequestTracker, opts...)
 	}
 }