diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go
index 7b10d78582..fd5daa198c 100644
--- a/pkg/controller/controllerutil/controllerutil.go
+++ b/pkg/controller/controllerutil/controllerutil.go
@@ -171,7 +171,14 @@ func RemoveControllerReference(owner, object metav1.Object, scheme *runtime.Sche
 	if ok := HasControllerReference(object); !ok {
 		return fmt.Errorf("%T does not have a owner reference with controller equals true", object)
 	}
-	return RemoveOwnerReference(owner, object, scheme)
+	ownerRefs := object.GetOwnerReferences()
+	for _, ownerRef := range ownerRefs {
+		t := ptr.To(true)
+		if ownerRef.Controller == t && ownerRef.Name == owner.GetName() {
+			return RemoveOwnerReference(owner, object, scheme)
+		}
+	}
+	return fmt.Errorf("%T is not owned reference by %T where controller equals true", object, owner)
 }
 
 func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) {