diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go
index 7b10d78582..9f68e7f380 100644
--- a/pkg/controller/controllerutil/controllerutil.go
+++ b/pkg/controller/controllerutil/controllerutil.go
@@ -142,6 +142,7 @@ func RemoveOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) e
 		APIVersion: gvk.GroupVersion().String(),
 		Name:       owner.GetName(),
 		Kind:       gvk.Kind,
+		Controller: ptr.To(true),
 	})
 	if index == -1 {
 		return fmt.Errorf("%T does not have an owner reference for %T", object, owner)
@@ -171,7 +172,26 @@ func RemoveControllerReference(owner, object metav1.Object, scheme *runtime.Sche
 	if ok := HasControllerReference(object); !ok {
 		return fmt.Errorf("%T does not have a owner reference with controller equals true", object)
 	}
-	return RemoveOwnerReference(owner, object, scheme)
+	ro, ok := owner.(runtime.Object)
+	if !ok {
+		return fmt.Errorf("%T is not a runtime.Object, cannot call RemoveControllerReference", owner)
+	}
+	gvk, err := apiutil.GVKForObject(ro, scheme)
+	if err != nil {
+		return err
+	}
+	ownerRefs := object.GetOwnerReferences()
+	index := indexOwnerRef(ownerRefs, metav1.OwnerReference{
+		APIVersion: gvk.GroupVersion().String(),
+		Name:       owner.GetName(),
+		Kind:       gvk.Kind,
+	})
+	if index == -1 {
+		return fmt.Errorf("%T does not have an controller reference for %T", object, owner)
+	}
+	ownerRefs = append(ownerRefs[:index], ownerRefs[index+1:]...)
+	object.SetOwnerReferences(ownerRefs)
+	return nil
 }
 
 func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) {