From 995b853e74c2e2c9c348a6711e76543205f1c8e1 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Fri, 1 Nov 2024 11:38:18 +0800 Subject: [PATCH] Enhanced forced deletion on karmadactl unjoin Signed-off-by: zhzhuang-zju --- pkg/karmadactl/unjoin/unjoin.go | 5 +- pkg/karmadactl/unregister/unregister.go | 19 +++++- pkg/karmadactl/util/cluster.go | 87 +++++++++++++++++++++++-- 3 files changed, 103 insertions(+), 8 deletions(-) diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go index 92faf95644c4..486911160451 100644 --- a/pkg/karmadactl/unjoin/unjoin.go +++ b/pkg/karmadactl/unjoin/unjoin.go @@ -147,7 +147,7 @@ func (j *CommandUnjoinOption) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&j.ClusterKubeConfig, "cluster-kubeconfig", "", "Path of the cluster's kubeconfig.") flags.BoolVar(&j.forceDeletion, "force", false, - "Delete cluster and secret resources even if resources in the cluster targeted for unjoin are not removed successfully.") + "If true, when the unjoin process fails due to a timeout, force unjoin the member cluster by removing the finalizer from the related work, namespace, and cluster resources. Note that forcing the unjoin of a member cluster may result in residual resources and requires confirmation.") flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.") flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") } @@ -181,9 +181,10 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error { // RunUnJoinCluster unJoin the cluster from karmada. func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error { controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) + controlPlaneKubeClient := kubeclient.NewForConfigOrDie(controlPlaneRestConfig) // delete the cluster object in host cluster that associates the unjoining cluster - err := cmdutil.DeleteClusterObject(controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun) + err := cmdutil.DeleteClusterObject(controlPlaneKubeClient, controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion) if err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go index f3361539c262..9ef5cfeedf1f 100644 --- a/pkg/karmadactl/unregister/unregister.go +++ b/pkg/karmadactl/unregister/unregister.go @@ -130,6 +130,9 @@ type CommandUnregisterOption struct { // ControlPlaneClient control plane client set ControlPlaneClient karmadaclientset.Interface + // ControlPlaneKubeClient control plane kube client set + ControlPlaneKubeClient kubeclient.Interface + // MemberClusterClient member cluster client set MemberClusterClient kubeclient.Interface } @@ -223,6 +226,10 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromFile() error { if err != nil { return fmt.Errorf("failed to build karmada control plane clientset: %w", err) } + j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg) + if err != nil { + return fmt.Errorf("failed to build kube control plane clientset: %w", err) + } return nil } @@ -248,7 +255,14 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromAgent() error { } j.ControlPlaneClient, err = register.ToKarmadaClient(karmadaCfg) - return err + if err != nil { + return fmt.Errorf("failed to build karmada control plane clientset: %w", err) + } + j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg) + if err != nil { + return fmt.Errorf("failed to build kube control plane clientset: %w", err) + } + return nil } func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment) (*clientcmdapi.Config, error) { @@ -305,7 +319,8 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { } // 1. delete the cluster object from the Karmada control plane - if err := cmdutil.DeleteClusterObject(j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun); err != nil { + //TODO: add flag --force to implement force deletaion. + if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } diff --git a/pkg/karmadactl/util/cluster.go b/pkg/karmadactl/util/cluster.go index ff578c63211c..d54809626ba5 100644 --- a/pkg/karmadactl/util/cluster.go +++ b/pkg/karmadactl/util/cluster.go @@ -24,14 +24,19 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + kubeclient "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" ) // DeleteClusterObject deletes the cluster object from the Karmada control plane. -func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string, - timeout time.Duration, dryRun bool) error { +// If the operation times out and `forceDeletion` is true, then force deletion begins, which involves sequentially deleting the `work`, `executionSpace`, and `cluster` finalizers. +func DeleteClusterObject(controlPlaneKubeClient kubeclient.Interface, controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string, + timeout time.Duration, dryRun bool, forceDeletion bool) error { if dryRun { return nil } @@ -58,10 +63,84 @@ func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, c klog.Infof("Waiting for the cluster object %s to be deleted", clusterName) return false, nil }) + + if err != nil && forceDeletion { + klog.Warningf("Deleting the cluster object timed out. cluster name: %s, error: %v", clusterName, err) + klog.Infof("Start forced deletion. cluster name: %s", clusterName) + executionSpaceName := names.GenerateExecutionSpaceName(clusterName) + err = removeWorkFinalizer(executionSpaceName, controlPlaneKarmadaClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Work, error: %v", err) + } + + err = removeExecutionSpaceFinalizer(executionSpaceName, controlPlaneKubeClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Namespace(%s), error: %v", executionSpaceName, err) + } + + err = removeClusterFinalizer(clusterName, controlPlaneKarmadaClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Cluster(%s), error: %v", clusterName, err) + } + + klog.Infof("Forced deletion is complete.") + return nil + } + + return err +} + +// removeWorkFinalizer removes the finalizer of works from the executionSpace +func removeWorkFinalizer(executionSpaceName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).List(context.TODO(), metav1.ListOptions{}) if err != nil { - klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err) - return err + return fmt.Errorf("failed to list work in executionSpace %s", executionSpaceName) } + for i := range list.Items { + work := &list.Items[i] + if !controllerutil.ContainsFinalizer(work, util.ExecutionControllerFinalizer) { + continue + } + controllerutil.RemoveFinalizer(work, util.ExecutionControllerFinalizer) + _, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).Update(context.TODO(), work, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to remove the finalizer of work(%s/%s)", executionSpaceName, work.GetName()) + } + } return nil } + +// removeExecutionSpaceFinalizer removes the finalizer of executionSpace +func removeExecutionSpaceFinalizer(executionSpaceName string, controlPlaneKubeClient kubeclient.Interface) error { + executionSpace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), executionSpaceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Namespace(%s)", executionSpaceName) + } + + if !controllerutil.ContainsFinalizer(executionSpace, "kubernetes") { + return nil + } + + controllerutil.RemoveFinalizer(executionSpace, "kubernetes") + _, err = controlPlaneKubeClient.CoreV1().Namespaces().Update(context.TODO(), executionSpace, metav1.UpdateOptions{}) + + return err +} + +// removeClusterFinalizer removes the finalizer of cluster object +func removeClusterFinalizer(clusterName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Cluster(%s)", clusterName) + } + + if !controllerutil.ContainsFinalizer(cluster, util.ClusterControllerFinalizer) { + return nil + } + + controllerutil.RemoveFinalizer(cluster, util.ClusterControllerFinalizer) + _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Update(context.TODO(), cluster, metav1.UpdateOptions{}) + + return err +}