Set ObservedGeneration in conditions and update tests

pkg/controller/admissionchecks/multikueue/admissioncheck.go

@@ -70,10 +70,11 @@ func (a *ACReconciler) Reconcile(ctx context.Context, req reconcile.Request) (re
 	log.V(2).Info("Reconcile AdmissionCheck")
 
 	newCondition := metav1.Condition{
-		Type:    kueue.AdmissionCheckActive,
-		Status:  metav1.ConditionTrue,
-		Reason:  "Active",
-		Message: "The admission check is active",
+		Type:               kueue.AdmissionCheckActive,
+		Status:             metav1.ConditionTrue,
+		Reason:             "Active",
+		Message:            "The admission check is active",
+		ObservedGeneration: ac.Generation,
 	}
 
 	if cfg, err := a.helper.ConfigFromRef(ctx, ac.Spec.Parameters); err != nil {

pkg/controller/admissionchecks/multikueue/multikueuecluster.go

@@ -454,10 +454,11 @@ func (c *clustersReconciler) getKubeConfigFromPath(path string) ([]byte, bool, e
 
 func (c *clustersReconciler) updateStatus(ctx context.Context, cluster *kueuealpha.MultiKueueCluster, active bool, reason, message string) error {
 	newCondition := metav1.Condition{
-		Type:    kueuealpha.MultiKueueClusterActive,
-		Status:  metav1.ConditionFalse,
-		Reason:  reason,
-		Message: message,
+		Type:               kueuealpha.MultiKueueClusterActive,
+		Status:             metav1.ConditionFalse,
+		Reason:             reason,
+		Message:            message,
+		ObservedGeneration: cluster.Generation,
 	}
 	if active {
 		newCondition.Status = metav1.ConditionTrue

pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler.go

@@ -43,10 +43,11 @@ func (a *acReconciler) Reconcile(ctx context.Context, req reconcile.Request) (re
 
 	currentCondition := ptr.Deref(apimeta.FindStatusCondition(ac.Status.Conditions, kueue.AdmissionCheckActive), metav1.Condition{})
 	newCondition := metav1.Condition{
-		Type:    kueue.AdmissionCheckActive,
-		Status:  metav1.ConditionTrue,
-		Reason:  "Active",
-		Message: "The admission check is active",
+		Type:               kueue.AdmissionCheckActive,
+		Status:             metav1.ConditionTrue,
+		Reason:             "Active",
+		Message:            "The admission check is active",
+		ObservedGeneration: ac.Generation,
 	}
 
 	if _, err := a.helper.ConfigFromRef(ctx, ac.Spec.Parameters); err != nil {

pkg/controller/core/clusterqueue_controller.go

@@ -650,10 +650,11 @@ func (r *ClusterQueueReconciler) updateCqStatusIfChanged(
 	cq.Status.PendingWorkloads = int32(pendingWorkloads)
 	cq.Status.PendingWorkloadsStatus = r.getWorkloadsStatus(cq)
 	meta.SetStatusCondition(&cq.Status.Conditions, metav1.Condition{
-		Type:    kueue.ClusterQueueActive,
-		Status:  conditionStatus,
-		Reason:  reason,
-		Message: msg,
+		Type:               kueue.ClusterQueueActive,
+		Status:             conditionStatus,
+		Reason:             reason,
+		Message:            msg,
+		ObservedGeneration: cq.Generation,
 	})
 	if !equality.Semantic.DeepEqual(cq.Status, oldStatus) {
 		return r.client.Status().Update(ctx, cq)

pkg/controller/core/localqueue_controller.go

@@ -288,10 +288,11 @@ func (r *LocalQueueReconciler) UpdateStatusIfChanged(
 	queue.Status.FlavorUsage = stats.AdmittedResources
 	if len(conditionStatus) != 0 && len(reason) != 0 && len(msg) != 0 {
 		meta.SetStatusCondition(&queue.Status.Conditions, metav1.Condition{
-			Type:    kueue.LocalQueueActive,
-			Status:  conditionStatus,
-			Reason:  reason,
-			Message: msg,
+			Type:               kueue.LocalQueueActive,
+			Status:             conditionStatus,
+			Reason:             reason,
+			Message:            msg,
+			ObservedGeneration: queue.Generation,
 		})
 	}
 	if !equality.Semantic.DeepEqual(oldStatus, queue.Status) {

pkg/controller/jobframework/reconciler.go

@@ -959,10 +959,11 @@ func generatePodsReadyCondition(job GenericJob, wl *kueue.Workload) metav1.Condi
 		message = "All pods were ready or succeeded since the workload admission"
 	}
 	return metav1.Condition{
-		Type:    kueue.WorkloadPodsReady,
-		Status:  conditionStatus,
-		Reason:  "PodsReady",
-		Message: message,
+		Type:               kueue.WorkloadPodsReady,
+		Status:             conditionStatus,
+		Reason:             "PodsReady",
+		Message:            message,
+		ObservedGeneration: wl.Generation,
 	}
 }
 

pkg/controller/jobs/job/job_controller.go

@@ -290,10 +290,11 @@ func (j *Job) Finished() (metav1.Condition, bool) {
 	}
 
 	condition := metav1.Condition{
-		Type:    kueue.WorkloadFinished,
-		Status:  metav1.ConditionTrue,
-		Reason:  "JobFinished",
-		Message: "Job finished successfully",
+		Type:               kueue.WorkloadFinished,
+		Status:             metav1.ConditionTrue,
+		Reason:             "JobFinished",
+		Message:            "Job finished successfully",
+		ObservedGeneration: j.Generation,
 	}
 	if conditionType == batchv1.JobFailed {
 		condition.Message = "Job failed"

pkg/controller/jobs/mpijob/mpijob_controller.go

@@ -164,10 +164,11 @@ func (j *MPIJob) Finished() (metav1.Condition, bool) {
 		message = "Job failed"
 	}
 	condition := metav1.Condition{
-		Type:    kueue.WorkloadFinished,
-		Status:  metav1.ConditionTrue,
-		Reason:  "JobFinished",
-		Message: message,
+		Type:               kueue.WorkloadFinished,
+		Status:             metav1.ConditionTrue,
+		Reason:             "JobFinished",
+		Message:            message,
+		ObservedGeneration: j.Generation,
 	}
 	return condition, finished
 }

pkg/controller/jobs/rayjob/rayjob_controller.go

@@ -162,10 +162,11 @@ func (j *RayJob) RestorePodSetsInfo(podSetsInfo []podset.PodSetInfo) bool {
 
 func (j *RayJob) Finished() (metav1.Condition, bool) {
 	condition := metav1.Condition{
-		Type:    kueue.WorkloadFinished,
-		Status:  metav1.ConditionTrue,
-		Reason:  string(j.Status.JobStatus),
-		Message: j.Status.Message,
+		Type:               kueue.WorkloadFinished,
+		Status:             metav1.ConditionTrue,
+		Reason:             string(j.Status.JobStatus),
+		Message:            j.Status.Message,
+		ObservedGeneration: j.Generation,
 	}
 
 	return condition, j.Status.JobStatus == rayjobapi.JobStatusFailed || j.Status.JobStatus == rayjobapi.JobStatusSucceeded

pkg/scheduler/scheduler_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"sigs.k8s.io/kueue/test/util"
 	"sort"
 	"sync"
 	"testing"
@@ -2051,8 +2052,6 @@ func TestLastSchedulingContext(t *testing.T) {
 	}
 }
 
-var ignoreConditionTimestamps = cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")
-
 func TestRequeueAndUpdate(t *testing.T) {
 	cq := utiltesting.MakeClusterQueue("cq").Obj()
 	q1 := utiltesting.MakeLocalQueue("q1", "ns1").ClusterQueue(cq.Name).Obj()
@@ -2181,7 +2180,7 @@ func TestRequeueAndUpdate(t *testing.T) {
 			if err := cl.Get(ctx, client.ObjectKeyFromObject(w1), &updatedWl); err != nil {
 				t.Fatalf("Failed obtaining updated object: %v", err)
 			}
-			if diff := cmp.Diff(tc.wantStatus, updatedWl.Status, ignoreConditionTimestamps); diff != "" {
+			if diff := cmp.Diff(tc.wantStatus, updatedWl.Status, util.IgnoreConditionTimestamps); diff != "" {
 				t.Errorf("Unexpected status after updating (-want,+got):\n%s", diff)
 			}
 			// Make sure a second call doesn't make unnecessary updates.

pkg/util/testing/wrappers.go

@@ -87,6 +87,11 @@ func (w *WorkloadWrapper) UID(uid types.UID) *WorkloadWrapper {
 	return w
 }
 
+func (w *WorkloadWrapper) Generation(num int64) *WorkloadWrapper {
+	w.ObjectMeta.Generation = num
+	return w
+}
+
 func (w *WorkloadWrapper) Name(name string) *WorkloadWrapper {
 	w.Workload.Name = name
 	return w

pkg/workload/admissionchecks.go

@@ -38,10 +38,11 @@ func SyncAdmittedCondition(w *kueue.Workload) bool {
 		return false
 	}
 	newCondition := metav1.Condition{
-		Type:    kueue.WorkloadAdmitted,
-		Status:  metav1.ConditionTrue,
-		Reason:  "Admitted",
-		Message: "The workload is admitted",
+		Type:               kueue.WorkloadAdmitted,
+		Status:             metav1.ConditionTrue,
+		Reason:             "Admitted",
+		Message:            "The workload is admitted",
+		ObservedGeneration: w.Generation,
 	}
 	switch {
 	case !hasReservation && !hasAllChecksReady:

pkg/workload/workload.go

@@ -311,6 +311,7 @@ func UpdateStatus(ctx context.Context,
 		LastTransitionTime: now,
 		Reason:             reason,
 		Message:            api.TruncateConditionMessage(message),
+		ObservedGeneration: wl.Generation,
 	}
 
 	newWl := BaseSSAWorkload(wl)
@@ -328,6 +329,7 @@ func UnsetQuotaReservationWithCondition(wl *kueue.Workload, reason, message stri
 		LastTransitionTime: metav1.Now(),
 		Reason:             reason,
 		Message:            api.TruncateConditionMessage(message),
+		ObservedGeneration: wl.Generation,
 	}
 	changed := apimeta.SetStatusCondition(&wl.Status.Conditions, condition)
 	if wl.Status.Admission != nil {
@@ -374,6 +376,7 @@ func SetQuotaReservation(w *kueue.Workload, admission *kueue.Admission) {
 		LastTransitionTime: metav1.Now(),
 		Reason:             "QuotaReserved",
 		Message:            fmt.Sprintf("Quota reserved in ClusterQueue %s", w.Status.Admission.ClusterQueue),
+		ObservedGeneration: w.Generation,
 	}
 	apimeta.SetStatusCondition(&w.Status.Conditions, admittedCond)
 
@@ -391,6 +394,7 @@ func SetEvictedCondition(w *kueue.Workload, reason string, message string) {
 		LastTransitionTime: metav1.Now(),
 		Reason:             reason,
 		Message:            message,
+		ObservedGeneration: w.Generation,
 	}
 	apimeta.SetStatusCondition(&w.Status.Conditions, condition)
 }

pkg/workload/workload_test.go

@@ -199,8 +199,6 @@ func TestNewInfo(t *testing.T) {
 	}
 }
 
-var ignoreConditionTimestamps = cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")
-
 func TestUpdateWorkloadStatus(t *testing.T) {
 	cases := map[string]struct {
 		oldStatus  kueue.WorkloadStatus
@@ -218,10 +216,11 @@ func TestUpdateWorkloadStatus(t *testing.T) {
 			wantStatus: kueue.WorkloadStatus{
 				Conditions: []metav1.Condition{
 					{
-						Type:    kueue.WorkloadQuotaReserved,
-						Status:  metav1.ConditionFalse,
-						Reason:  "Pending",
-						Message: "didn't fit",
+						Type:               kueue.WorkloadQuotaReserved,
+						Status:             metav1.ConditionFalse,
+						Reason:             "Pending",
+						Message:            "didn't fit",
+						ObservedGeneration: 1,
 					},
 				},
 			},
@@ -243,17 +242,18 @@ func TestUpdateWorkloadStatus(t *testing.T) {
 			wantStatus: kueue.WorkloadStatus{
 				Conditions: []metav1.Condition{
 					{
-						Type:   kueue.WorkloadQuotaReserved,
-						Status: metav1.ConditionTrue,
-						Reason: "Admitted",
+						Type:               kueue.WorkloadQuotaReserved,
+						Status:             metav1.ConditionTrue,
+						Reason:             "Admitted",
+						ObservedGeneration: 1,
 					},
 				},
 			},
 		},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			workload := utiltesting.MakeWorkload("foo", "bar").Obj()
+			workload := utiltesting.MakeWorkload("foo", "bar").Generation(1).Obj()
 			workload.Status = tc.oldStatus
 			cl := utiltesting.NewFakeClientSSAAsSM(workload)
 			ctx := context.Background()
@@ -265,7 +265,7 @@ func TestUpdateWorkloadStatus(t *testing.T) {
 			if err := cl.Get(ctx, client.ObjectKeyFromObject(workload), &updatedWl); err != nil {
 				t.Fatalf("Failed obtaining updated object: %v", err)
 			}
-			if diff := cmp.Diff(tc.wantStatus, updatedWl.Status, ignoreConditionTimestamps); diff != "" {
+			if diff := cmp.Diff(tc.wantStatus, updatedWl.Status, util.IgnoreConditionTimestamps); diff != "" {
 				t.Errorf("Unexpected status after updating (-want,+got):\n%s", diff)
 			}
 		})

test/e2e/multikueue/e2e_test.go

@@ -18,6 +18,7 @@ package mke2e
 
 import (
 	"os/exec"
+	"sigs.k8s.io/kueue/test/util"
 
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/onsi/ginkgo/v2"
@@ -40,7 +41,6 @@ import (
 	testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job"
 	testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset"
 	"sigs.k8s.io/kueue/pkg/workload"
-	"sigs.k8s.io/kueue/test/util"
 )
 
 // +kubebuilder:docs-gen:collapse=Imports
@@ -221,7 +221,7 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 						Status:  metav1.ConditionTrue,
 						Reason:  "JobFinished",
 						Message: `Job finished successfully`,
-					}, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")))
+					}, util.IgnoreConditionTimestamps, util.IgnoreConditionObservedGeneration))
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
 
@@ -286,7 +286,7 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 						Status:  metav1.ConditionTrue,
 						Reason:  "Admitted",
 						Message: "The workload is admitted",
-					}, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")))
+					}, util.IgnoreConditionTimestamps, util.IgnoreConditionObservedGeneration))
 				}, util.Timeout, util.Interval).Should(gomega.Succeed())
 			})
 
@@ -314,7 +314,7 @@ var _ = ginkgo.Describe("MultiKueue", func() {
 						Status:  metav1.ConditionTrue,
 						Reason:  "JobSetFinished",
 						Message: "JobSet finished successfully",
-					}, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")))
+					}, util.IgnoreConditionTimestamps, util.IgnoreConditionObservedGeneration))
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
 

test/e2e/singlecluster/e2e_test.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kueue/test/util"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 	"sigs.k8s.io/kueue/pkg/controller/constants"
@@ -35,7 +36,6 @@ import (
 	"sigs.k8s.io/kueue/pkg/util/testing"
 	testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job"
 	"sigs.k8s.io/kueue/pkg/workload"
-	"sigs.k8s.io/kueue/test/util"
 )
 
 // +kubebuilder:docs-gen:collapse=Imports
@@ -207,7 +207,7 @@ var _ = ginkgo.Describe("Kueue", func() {
 							Type:   kueue.WorkloadFinished,
 							Status: metav1.ConditionTrue,
 							Reason: "JobFinished",
-						}, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message"))))
+						}, util.IgnoreConditionMessage, util.IgnoreConditionTimestamps, util.IgnoreConditionObservedGeneration)))
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
 		})

test/e2e/singlecluster/jobset_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package e2e
 
 import (
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
@@ -107,7 +106,7 @@ var _ = ginkgo.Describe("Kueue", func() {
 						Status:  metav1.ConditionTrue,
 						Reason:  "JobSetFinished",
 						Message: "JobSet finished successfully",
-					}, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")))
+					}, util.IgnoreConditionTimestamps, util.IgnoreConditionObservedGeneration))
 				}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 			})
 		})