From 3605624a58f16c0063014a5d7ccec130e894df6b Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Tue, 29 Oct 2024 19:05:05 +0600 Subject: [PATCH] temp Signed-off-by: souravbiswassanto --- apis/apps/fuzzer/fuzzer.go | 1 - apis/apps/install/install.go | 1 - apis/apps/register.go | 6 - apis/apps/v1alpha1/openapi_generated.go | 8 +- apis/apps/v1alpha1/sidekick_types.go | 7 +- client/clientset/versioned/clientset.go | 3 +- .../versioned/fake/clientset_generated.go | 7 +- client/clientset/versioned/fake/register.go | 3 +- client/clientset/versioned/scheme/register.go | 3 +- .../typed/apps/v1alpha1/apps_client.go | 3 +- .../apps/v1alpha1/fake/fake_apps_client.go | 3 +- .../typed/apps/v1alpha1/fake/fake_sidekick.go | 3 +- .../versioned/typed/apps/v1alpha1/sidekick.go | 5 +- crds/apps.k8s.appscode.com_sidekicks.yaml | 9 +- pkg/controllers/apps/sidekick_controller.go | 119 ++++++++++++------ 15 files changed, 107 insertions(+), 74 deletions(-) diff --git a/apis/apps/fuzzer/fuzzer.go b/apis/apps/fuzzer/fuzzer.go index 4d352d3c..8da1a48c 100644 --- a/apis/apps/fuzzer/fuzzer.go +++ b/apis/apps/fuzzer/fuzzer.go @@ -17,7 +17,6 @@ limitations under the License. package fuzzer import ( - "kubeops.dev/sidekick/apis/apps/v1alpha1" fuzz "github.com/google/gofuzz" diff --git a/apis/apps/install/install.go b/apis/apps/install/install.go index 6152319f..efa577a6 100644 --- a/apis/apps/install/install.go +++ b/apis/apps/install/install.go @@ -17,7 +17,6 @@ limitations under the License. package install import ( - "kubeops.dev/sidekick/apis/apps/v1alpha1" "k8s.io/apimachinery/pkg/runtime" diff --git a/apis/apps/register.go b/apis/apps/register.go index 96114eec..a350e2cb 100644 --- a/apis/apps/register.go +++ b/apis/apps/register.go @@ -19,9 +19,3 @@ package apps const ( GroupName = "apps.k8s.appscode.com" ) - - - - - - diff --git a/apis/apps/v1alpha1/openapi_generated.go b/apis/apps/v1alpha1/openapi_generated.go index b0ebb832..efa14694 100644 --- a/apis/apps/v1alpha1/openapi_generated.go +++ b/apis/apps/v1alpha1/openapi_generated.go @@ -19592,13 +19592,11 @@ func schema_sidekick_apis_apps_v1alpha1_LeaderStatus(ref common.ReferenceCallbac Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Type: []string{"string"}, + Format: "", }, }, }, - Required: []string{"name"}, }, }, } @@ -20134,7 +20132,6 @@ func schema_sidekick_apis_apps_v1alpha1_SidekickStatus(ref common.ReferenceCallb "pod": { SchemaProps: spec.SchemaProps{ Description: "Possible enum values:\n - `\"Failed\"` means that all containers in the pod have terminated, and at least one container has terminated in a failure (exited with a non-zero exit code or was stopped by the system).\n - `\"Pending\"` means the pod has been accepted by the system, but one or more of the containers has not been started. This includes time before being bound to a node, as well as time spent pulling images onto the host.\n - `\"Running\"` means the pod has been bound to a node and all of the containers have been started. At least one container is still running or is in the process of being restarted.\n - `\"Succeeded\"` means that all containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers.\n - `\"Unknown\"` means that for some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod. Deprecated: It isn't being set since 2015 (74da3b14b0c0f658b3bb8d2def5094686d0e9095)", - Default: "", Type: []string{"string"}, Format: "", Enum: []interface{}{"Failed", "Pending", "Running", "Succeeded", "Unknown"}, @@ -20201,7 +20198,6 @@ func schema_sidekick_apis_apps_v1alpha1_SidekickStatus(ref common.ReferenceCallb }, }, }, - Required: []string{"leader", "pod"}, }, }, Dependencies: []string{ diff --git a/apis/apps/v1alpha1/sidekick_types.go b/apis/apps/v1alpha1/sidekick_types.go index bf9eeccf..934841aa 100644 --- a/apis/apps/v1alpha1/sidekick_types.go +++ b/apis/apps/v1alpha1/sidekick_types.go @@ -114,6 +114,7 @@ type SidekickSpec struct { // terminated once the job backoff limit has been reached. This can make debugging the Job's executable // more difficult. We suggest setting restartPolicy = "Never" when debugging the Job or using a logging // system to ensure output from failed Jobs is not lost inadvertently. + // +kubebuilder:validation:Enum=Never;Always;OnFailure // +optional RestartPolicy core.RestartPolicy `json:"restartPolicy,omitempty"` // Specifies the number of retries before marking this job failed. @@ -537,13 +538,13 @@ type VolumeMount struct { } type LeaderStatus struct { - Name string `json:"name"` + Name string `json:"name,omitempty"` } // SidekickStatus defines the observed state of Sidekick type SidekickStatus struct { - Leader LeaderStatus `json:"leader"` - Pod core.PodPhase `json:"pod"` + Leader LeaderStatus `json:"leader,omitempty"` + Pod core.PodPhase `json:"pod,omitempty"` // Specifies the current phase of the sidekick CR // +optional diff --git a/client/clientset/versioned/clientset.go b/client/clientset/versioned/clientset.go index 05d0f62d..7c3f7fda 100644 --- a/client/clientset/versioned/clientset.go +++ b/client/clientset/versioned/clientset.go @@ -22,10 +22,11 @@ import ( "fmt" "net/http" + appsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" + discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" - appsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" ) type Interface interface { diff --git a/client/clientset/versioned/fake/clientset_generated.go b/client/clientset/versioned/fake/clientset_generated.go index da3739bf..17445e82 100644 --- a/client/clientset/versioned/fake/clientset_generated.go +++ b/client/clientset/versioned/fake/clientset_generated.go @@ -19,14 +19,15 @@ limitations under the License. package fake import ( + clientset "kubeops.dev/sidekick/client/clientset/versioned" + appsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" + fakeappsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1/fake" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" - clientset "kubeops.dev/sidekick/client/clientset/versioned" - appsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" - fakeappsv1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. diff --git a/client/clientset/versioned/fake/register.go b/client/clientset/versioned/fake/register.go index f3dcb527..429d7875 100644 --- a/client/clientset/versioned/fake/register.go +++ b/client/clientset/versioned/fake/register.go @@ -19,12 +19,13 @@ limitations under the License. package fake import ( + appsv1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - appsv1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" ) var scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/scheme/register.go b/client/clientset/versioned/scheme/register.go index 2a21dabe..9b240855 100644 --- a/client/clientset/versioned/scheme/register.go +++ b/client/clientset/versioned/scheme/register.go @@ -19,12 +19,13 @@ limitations under the License. package scheme import ( + appsv1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - appsv1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" ) var Scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go b/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go index 9135d861..68369996 100644 --- a/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go +++ b/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go @@ -21,9 +21,10 @@ package v1alpha1 import ( "net/http" - rest "k8s.io/client-go/rest" v1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" "kubeops.dev/sidekick/client/clientset/versioned/scheme" + + rest "k8s.io/client-go/rest" ) type AppsV1alpha1Interface interface { diff --git a/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go b/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go index 87f9e943..9596d7fd 100644 --- a/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go +++ b/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go @@ -19,9 +19,10 @@ limitations under the License. package fake import ( + v1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" + rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" - v1alpha1 "kubeops.dev/sidekick/client/clientset/versioned/typed/apps/v1alpha1" ) type FakeAppsV1alpha1 struct { diff --git a/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_sidekick.go b/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_sidekick.go index 08c9f64b..0958c9f3 100644 --- a/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_sidekick.go +++ b/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_sidekick.go @@ -21,12 +21,13 @@ package fake import ( "context" + v1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" - v1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" ) // FakeSidekicks implements SidekickInterface diff --git a/client/clientset/versioned/typed/apps/v1alpha1/sidekick.go b/client/clientset/versioned/typed/apps/v1alpha1/sidekick.go index 06db1681..083a939b 100644 --- a/client/clientset/versioned/typed/apps/v1alpha1/sidekick.go +++ b/client/clientset/versioned/typed/apps/v1alpha1/sidekick.go @@ -22,12 +22,13 @@ import ( "context" "time" + v1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" + scheme "kubeops.dev/sidekick/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" - v1alpha1 "kubeops.dev/sidekick/apis/apps/v1alpha1" - scheme "kubeops.dev/sidekick/client/clientset/versioned/scheme" ) // SidekicksGetter has a method to return a SidekickInterface. diff --git a/crds/apps.k8s.appscode.com_sidekicks.yaml b/crds/apps.k8s.appscode.com_sidekicks.yaml index d45b745e..0ce1de32 100644 --- a/crds/apps.k8s.appscode.com_sidekicks.yaml +++ b/crds/apps.k8s.appscode.com_sidekicks.yaml @@ -5411,6 +5411,10 @@ spec: more difficult. We suggest setting restartPolicy = "Never" when debugging the Job or using a logging system to ensure output from failed Jobs is not lost inadvertently.' + enum: + - Never + - Always + - OnFailure type: string runtimeClassName: description: 'RuntimeClassName refers to a RuntimeClass object in @@ -7686,8 +7690,6 @@ spec: properties: name: type: string - required: - - name type: object observedGeneration: description: observedGeneration is the most recent generation observed @@ -7707,9 +7709,6 @@ spec: description: PodPhase is a label for the condition of a pod at the current time. type: string - required: - - leader - - pod type: object type: object served: true diff --git a/pkg/controllers/apps/sidekick_controller.go b/pkg/controllers/apps/sidekick_controller.go index 194a35e7..dd9fa1f7 100644 --- a/pkg/controllers/apps/sidekick_controller.go +++ b/pkg/controllers/apps/sidekick_controller.go @@ -50,11 +50,16 @@ import ( ) const ( - keyHash = "sidekick.appscode.com/hash" - keyLeader = "sidekick.appscode.com/leader" - podHash = "sidekick.appscode.com/pod-specific-hash" + keyHash = "sidekick.appscode.com/hash" + keyLeader = "sidekick.appscode.com/leader" + podHash = "sidekick.appscode.com/pod-specific-hash" + finalizerSuffix = "finalizer" ) +func getFinalizerName() string { + return appsv1alpha1.SchemeGroupVersion.Group + "/" + finalizerSuffix +} + // SidekickReconciler reconciles a Sidekick object type SidekickReconciler struct { client.Client @@ -78,6 +83,25 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c klog.Infoln(fmt.Sprintf("reconciling %v ", req.NamespacedName)) logger := log.FromContext(ctx, "sidekick", req.Name, "ns", req.Namespace) ctx = log.IntoContext(ctx, logger) + + var pod corev1.Pod + err := r.Get(ctx, req.NamespacedName, &pod) + if err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil && pod.DeletionTimestamp != nil { + // removing finalizer, the reason behind adding this finalizer is stated below + // where we created the pod + if core_util.HasFinalizer(pod.ObjectMeta, getFinalizerName()) { + err = r.removeFinalizer(ctx, &pod) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + var sidekick appsv1alpha1.Sidekick if err := r.Get(ctx, req.NamespacedName, &sidekick); err != nil { logger.Error(err, "unable to fetch Sidekick") @@ -88,28 +112,24 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } if sidekick.DeletionTimestamp != nil { - if core_util.HasFinalizer(sidekick.ObjectMeta, appsv1alpha1.SchemeGroupVersion.Group) { + if core_util.HasFinalizer(sidekick.ObjectMeta, getFinalizerName()) { return ctrl.Result{}, r.terminate(ctx, &sidekick) } } + klog.Infoln("getting fin", sidekick.ObjectMeta.GetFinalizers()) - _, err := cu.CreateOrPatch(context.TODO(), r.Client, &sidekick, + _, err = cu.CreateOrPatch(context.TODO(), r.Client, &sidekick, func(in client.Object, createOp bool) client.Object { sk := in.(*appsv1alpha1.Sidekick) - sk.ObjectMeta = core_util.AddFinalizer(sk.ObjectMeta, appsv1alpha1.SchemeGroupVersion.Group) - + sk.ObjectMeta = core_util.AddFinalizer(sk.ObjectMeta, getFinalizerName()) return sk }, ) + klog.Infoln("getting fin down", sidekick.ObjectMeta.GetFinalizers()) klog.Infoln("1") if err != nil { return ctrl.Result{}, err } - err = r.Get(ctx, req.NamespacedName, &sidekick) - if err != nil { - return ctrl.Result{}, err - } - klog.Infoln("2") phase, err := r.sidekickStatus(ctx, &sidekick) if err != nil && !errors.IsNotFound(err) { @@ -148,7 +168,7 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c e2 := r.Get(ctx, req.NamespacedName, &pod) if e2 == nil { - err := r.Delete(ctx, &pod) + err := r.deletePod(ctx, &pod) if err != nil { return ctrl.Result{}, err } @@ -170,8 +190,6 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } else if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - - var pod corev1.Pod e2 := r.Get(ctx, req.NamespacedName, &pod) if e2 != nil && !errors.IsNotFound(e2) { return ctrl.Result{}, client.IgnoreNotFound(e2) @@ -190,7 +208,7 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if expectedHash != actualHash || leader.Name != pod.Annotations[keyLeader] || leader.Spec.NodeName != pod.Spec.NodeName || (pod.Status.Phase == corev1.PodFailed && sidekick.Spec.RestartPolicy == corev1.RestartPolicyNever) { - err := r.Delete(ctx, &pod) + err := r.deletePod(ctx, &pod) if err != nil { return ctrl.Result{}, err } @@ -277,10 +295,10 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } curTime := time.Now().String() curTimeHash := sha256.Sum256([]byte(curTime)) - pod.Annotations[podHash] = hex.EncodeToString(curTimeHash[:]) + pod.Annotations[podHash] = hex.EncodeToString(curTimeHash[:])[:16] pod.Annotations[keyHash] = meta.GenerationHash(&sidekick) pod.Annotations[keyLeader] = leader.Name - sidekick.Annotations[podHash] = hex.EncodeToString(curTimeHash[:]) + sidekick.Annotations[podHash] = hex.EncodeToString(curTimeHash[:])[:16] klog.Infoln("----------------------------------------------------------") for _, c := range sidekick.Spec.Containers { c2, err := convContainer(leader, c) @@ -296,8 +314,18 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } pod.Spec.InitContainers = append(pod.Spec.InitContainers, *c2) } + // Adding finalizer to pod because when user will delete this pod using + // kubectl delete, then pod will be gracefully terminated which will led + // to pod.status.phase: succeeded. We need to control this behaviour. + // By adding finalizer, we will know who is deleting the object klog.Infoln("hello ") - e3 := r.Create(ctx, &pod) + _, e3 := cu.CreateOrPatch(context.TODO(), r.Client, &pod, + func(in client.Object, createOp bool) client.Object { + po := in.(*corev1.Pod) + po.ObjectMeta = core_util.AddFinalizer(po.ObjectMeta, getFinalizerName()) + return po + }, + ) klog.Infoln("hello from down") if e3 != nil { @@ -307,7 +335,7 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c _, err = cu.CreateOrPatch(context.TODO(), r.Client, &sidekick, func(in client.Object, createOp bool) client.Object { sk := in.(*appsv1alpha1.Sidekick) - sk.Annotations[podHash] = hex.EncodeToString(curTimeHash[:]) + sk.Annotations[podHash] = hex.EncodeToString(curTimeHash[:])[:16] return sk }, ) @@ -332,7 +360,6 @@ func (r *SidekickReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } func (r *SidekickReconciler) updateSidekickStatus(ctx context.Context, sidekick *appsv1alpha1.Sidekick) error { - _, err := cu.PatchStatus(ctx, r.Client, sidekick, func(obj client.Object) client.Object { sk := obj.(*appsv1alpha1.Sidekick) sk.Status = sidekick.Status @@ -544,7 +571,7 @@ func (r *SidekickReconciler) terminate(ctx context.Context, sidekick *appsv1alph _, err = cu.CreateOrPatch(context.TODO(), r.Client, sidekick, func(in client.Object, createOp bool) client.Object { sk := in.(*appsv1alpha1.Sidekick) - sk.ObjectMeta = core_util.RemoveFinalizer(sk.ObjectMeta, appsv1alpha1.SchemeGroupVersion.Group) + sk.ObjectMeta = core_util.RemoveFinalizer(sk.ObjectMeta, getFinalizerName()) return sk }, @@ -552,28 +579,12 @@ func (r *SidekickReconciler) terminate(ctx context.Context, sidekick *appsv1alph return err } -func (r *SidekickReconciler) removeRestartCountForPod(ctx context.Context, sidekick *appsv1alpha1.Sidekick, pod *corev1.Pod) error { - hash, exists := pod.Annotations[podHash] - if !exists { - return nil - } - _, exists = sidekick.Status.ContainerRestartCountsPerPod[hash] - if !exists { - return nil - } - delete(sidekick.Status.ContainerRestartCountsPerPod, hash) - err := r.Status().Update(ctx, sidekick) - if err != nil { - return err - } - return nil -} - func (r *SidekickReconciler) sidekickStatus(ctx context.Context, sidekick *appsv1alpha1.Sidekick) (appsv1alpha1.SideKickPhase, error) { var pod corev1.Pod err := r.Get(ctx, client.ObjectKeyFromObject(sidekick), &pod) if err != nil { - return "", err + klog.Infoln("phase checking -----", sidekick.Status.Phase) + return sidekick.Status.Phase, err } podSpecificHash := pod.Annotations[podHash] restartCounter := getContainerRestartCounts(&pod) @@ -605,9 +616,13 @@ func (r *SidekickReconciler) getSidekickPhase(sidekick *appsv1alpha1.Sidekick, p // now restartPolicy onFailure & Never remaining // In both cases we return phase as succeeded if our // pod return with exit code 0 - if pod.Status.Phase == corev1.PodSucceeded { + if pod.Status.Phase == corev1.PodSucceeded && pod.ObjectMeta.DeletionTimestamp == nil { return appsv1alpha1.SidekickPhaseSucceeded } + if sidekick.Status.Phase != appsv1alpha1.SidekickPhaseSucceeded && pod.Status.Phase == corev1.PodSucceeded && pod.ObjectMeta.DeletionTimestamp != nil { + podSpecificHash := pod.Annotations[podHash] + sidekick.Status.FailureCount[podSpecificHash] = true + } // Now we will figure if we should update the sidekick phase // as failed or not by checking the backOffLimit @@ -668,3 +683,25 @@ func (r *SidekickReconciler) updatePodFailureCount(ctx context.Context, sidekick sidekick.Status.FailureCount[hash] = true return r.updateSidekickStatus(ctx, sidekick) } + +func (r *SidekickReconciler) deletePod(ctx context.Context, pod *corev1.Pod) error { + err := r.removeFinalizer(ctx, pod) + if err != nil { + return err + } + return r.Delete(ctx, pod) +} + +func (r *SidekickReconciler) removeFinalizer(ctx context.Context, pod *corev1.Pod) error { + klog.Infoln("here in removing finalizer") + _, err := cu.CreateOrPatch(ctx, r.Client, pod, + func(in client.Object, createOp bool) client.Object { + po := in.(*corev1.Pod) + po.ObjectMeta = core_util.RemoveFinalizer(po.ObjectMeta, getFinalizerName()) + return po + }, + ) + klog.Infoln("here in removing finalizer done") + + return err +}