diff --git a/pkg/controllers/jobset_controller.go b/pkg/controllers/jobset_controller.go
index 21ab304f0..f93ad4500 100644
--- a/pkg/controllers/jobset_controller.go
+++ b/pkg/controllers/jobset_controller.go
@@ -550,8 +550,8 @@ func labelAndAnnotateObject(obj metav1.Object, js *jobset.JobSet, rjob *jobset.R
 	labels[jobset.JobIndexKey] = strconv.Itoa(jobIdx)
 
 	annotations := cloneMap(obj.GetAnnotations())
-	labels[jobset.JobSetNameKey] = js.Name
-	labels[jobset.ReplicatedJobNameKey] = rjob.Name
+	annotations[jobset.JobSetNameKey] = js.Name
+	annotations[jobset.ReplicatedJobNameKey] = rjob.Name
 	annotations[jobset.ReplicatedJobReplicas] = strconv.Itoa(rjob.Replicas)
 	annotations[jobset.JobIndexKey] = strconv.Itoa(jobIdx)
 
diff --git a/pkg/controllers/jobset_controller_test.go b/pkg/controllers/jobset_controller_test.go
index 888e5d581..5ec71abc3 100644
--- a/pkg/controllers/jobset_controller_test.go
+++ b/pkg/controllers/jobset_controller_test.go
@@ -720,6 +720,8 @@ func makeJob(args *makeJobArgs) *testutils.JobWrapper {
 			RestartsKey:                  strconv.Itoa(args.restarts),
 		}).
 		JobAnnotations(map[string]string{
+			jobset.JobSetNameKey:         args.jobSetName,
+			jobset.ReplicatedJobNameKey:  args.replicatedJobName,
 			jobset.ReplicatedJobReplicas: strconv.Itoa(args.replicas),
 			jobset.JobIndexKey:           strconv.Itoa(args.jobIdx),
 		}).
@@ -731,6 +733,8 @@ func makeJob(args *makeJobArgs) *testutils.JobWrapper {
 			RestartsKey:                  strconv.Itoa(args.restarts),
 		}).
 		PodAnnotations(map[string]string{
+			jobset.JobSetNameKey:         args.jobSetName,
+			jobset.ReplicatedJobNameKey:  args.replicatedJobName,
 			jobset.ReplicatedJobReplicas: strconv.Itoa(args.replicas),
 			jobset.JobIndexKey:           strconv.Itoa(args.jobIdx),
 		})