diff --git a/changelogs/unreleased/6155-kaovilai b/changelogs/unreleased/6155-kaovilai new file mode 100644 index 0000000000..bab83d4bfe --- /dev/null +++ b/changelogs/unreleased/6155-kaovilai @@ -0,0 +1 @@ +velero can now via install options, uninstall velero without deleting namespace \ No newline at end of file diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 722473b4f8..946b61ee28 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -79,6 +79,8 @@ type InstallOptions struct { Features string DefaultVolumesToFsBackup bool UploaderType string + Uninstall bool + PreserveUninstallNamespace bool } // BindFlags adds command line values to the options struct. @@ -118,6 +120,8 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the node-agent daemonset, if node-agent is enabled") flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.") flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, the supported values are '%s', '%s'", uploader.ResticType, uploader.KopiaType)) + flags.BoolVar(&o.Uninstall, "uninstall", o.Uninstall, "Uninstall Velero from the cluster. Optional.") + flags.BoolVar(&o.PreserveUninstallNamespace, "preserve-uninstall-namespace", o.PreserveUninstallNamespace, "Preserve the namespace used for uninstalling Velero. Optional.") } // NewInstallOptions instantiates a new, default InstallOptions struct. @@ -294,6 +298,14 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { if err != nil { return err } + if o.Uninstall { + if err := install.Uninstall(dynamicFactory, resources, os.Stdout, o.PreserveUninstallNamespace); err != nil { + return err + } else { + fmt.Println("Velero is uninstalled!") + return nil + } + } errorMsg := fmt.Sprintf("\n\nError installing Velero. Use `kubectl logs deploy/velero -n %s` to check the deploy logs", o.Namespace) err = install.Install(dynamicFactory, kbClient, resources, os.Stdout) diff --git a/pkg/cmd/cli/install/uninstall_test.go b/pkg/cmd/cli/install/uninstall_test.go new file mode 100644 index 0000000000..62e230ffb7 --- /dev/null +++ b/pkg/cmd/cli/install/uninstall_test.go @@ -0,0 +1,306 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + "context" + "fmt" + "net/url" + "os" + "testing" + "time" + + "github.com/spf13/cobra" + corev1api "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + kcapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" + velerov1clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + "github.com/vmware-tanzu/velero/pkg/install" +) + +func TestInstallUninstallOptions_Run(t *testing.T) { + t.Parallel() + type fields struct { + Uninstall bool + PreserveUninstallNamespace bool + ExpectedResourceCount int + } + tests := []struct { + name string + fields fields + wantErr bool + wantAnnotations map[string]string + wantLabels map[string]string + }{ + { + name: "Installed velero removed cleanly with uninstall flag", + fields: fields{ + Uninstall: true, + PreserveUninstallNamespace: false, + ExpectedResourceCount: 17, + }, + wantErr: false, + }, + { + name: "Installed velero removed but namespace is kept with uninstall preserveNamespace flag", + fields: fields{ + Uninstall: true, + PreserveUninstallNamespace: true, + ExpectedResourceCount: 17, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := NewInstallOptions() + env := envtest.Environment{} + cfg, err := env.Start() + if err != nil { + t.Errorf("Failed to start test environment: %v", err) + } + defer func() { + if err := env.Stop(); err != nil { + t.Errorf("Failed to stop test environment: %v", err) + } + }() + kubeconfigBytes, err := KubeConfigFromREST(cfg) + if err != nil { + t.Errorf("Failed to generate kubeconfig: %v", err) + } + if os.Getenv("KUBECONFIG") != "" { + panic(fmt.Sprintf("Unexpected KUBECONFIG is already set to %s", os.Getenv("KUBECONFIG"))) + } + envKubeConfigFileName := kubeConfigBytesToTmpFile(t, kubeconfigBytes) + defer func() { + if err := os.Remove(envKubeConfigFileName); err != nil { + t.Errorf("Failed to remove temp file: %v", err) + } + }() + fmt.Printf("envKubeConfigFileName: %s\n", envKubeConfigFileName) + os.Setenv("KUBECONFIG", envKubeConfigFileName) + defer func() { + if err := os.Unsetenv("KUBECONFIG"); err != nil { + t.Errorf("Failed to unset KUBECONFIG: %v", err) + } + }() + + veleroConfig, err := client.LoadConfig() + if err != nil { + t.Errorf("Failed to load velero config: %v", err) + } + f := client.NewFactory("test", veleroConfig) + if err := o.Run(&cobra.Command{}, f); (err != nil) != tt.wantErr { + t.Errorf("InstallOptions.Run() error = %v, wantErr %v", err, tt.wantErr) + } + clientset, err := f.KubeClient() + if err != nil { + t.Errorf("Failed to get kube client: %v", err) + } + success, count := countInstalledResources(clientset, cfg, t) + if !success { + t.Error("Error counting installed resources", count, tt.fields.ExpectedResourceCount) + } + if count != tt.fields.ExpectedResourceCount { + t.Errorf("Install did not create the expected number of resources. Expected %d, got %d", tt.fields.ExpectedResourceCount, count) + } + if tt.fields.Uninstall { + // now we run install with the uninstall flag + o.Uninstall = true + o.PreserveUninstallNamespace = tt.fields.PreserveUninstallNamespace + if err := o.Run(&cobra.Command{}, f); (err != nil) != tt.wantErr { + t.Errorf("InstallOptions.Run() error = %v, wantErr %v", err, tt.wantErr) + } + time.Sleep(2 * time.Second) + success, count := countInstalledResources(clientset, cfg, t) + if !success { + t.Error("Error counting installed resources") + } + if count != 0 && !tt.fields.PreserveUninstallNamespace { + t.Errorf("Uninstall did not remove all resources. Expected 0, got %d", count) + } + if tt.fields.PreserveUninstallNamespace { + if count != 1 { + t.Errorf("Uninstall did not preserve namespace. Expected 1, got %d", count) + } + // put dummy secret in namespace to test that it is preserved + secret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-secret", + Namespace: velerov1.DefaultNamespace, + }, + Data: map[string][]byte{ + "dummy-key": []byte("dummy-val"), + }, + } + _, err := clientset.CoreV1().Secrets(velerov1.DefaultNamespace).Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + t.Errorf("Failed to create dummy secret in namespace that supposedly still exists: %v", err) + } + } + } + }) + } +} + +func countInstalledResources(clientset kubernetes.Interface, cfg *rest.Config, t *testing.T) (bool, int) { + count := 0 + apiextensionsv1client := apiextensionsv1.NewForConfigOrDie(cfg) + veleroV1Client := velerov1clientset.NewForConfigOrDie(cfg) + crds, err := apiextensionsv1client.CustomResourceDefinitions().List(context.Background(), metav1.ListOptions{ + LabelSelector: labels.FormatLabels(install.Labels()), + }) + if err != nil { + t.Errorf("Failed to list CRDs: %v", err) + } + count += len(crds.Items) + for _, crd := range crds.Items { + fmt.Printf("count CRD: %s\n", crd.Name) + } + ns, err := clientset.CoreV1().Namespaces().Get(context.Background(), velerov1.DefaultNamespace, metav1.GetOptions{}) + if err == nil { + if ns.Status.Phase == corev1api.NamespaceActive { + // Quirks of envtest, the namespace is never removed, but it is marked as Terminating + // https://github.com/kubernetes-sigs/controller-runtime/issues/880 + count++ + } + } else if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Failed to get namespace: %v", err) + } + crbs, err := clientset.RbacV1().ClusterRoleBindings().List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Errorf("Failed to list cluster role bindings: %v", err) + } + for _, crb := range crbs.Items { + if len(crb.Subjects) > 0 && crb.Subjects[0].Name == "velero" { + if crb.Subjects[0].Namespace == velerov1.DefaultNamespace { + count++ + fmt.Printf("count CRB: %s\n", crb.Name) + break + } + } + } + _, err = clientset.CoreV1().ServiceAccounts(velerov1.DefaultNamespace).Get(context.Background(), "velero", metav1.GetOptions{}) + if err == nil { + count++ + } else if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Failed to get service account: %v", err) + } + _, err = veleroV1Client.BackupStorageLocations(velerov1.DefaultNamespace).Get(context.Background(), "default", metav1.GetOptions{}) + if err == nil { + count++ + } else if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Failed to get backup storage location: %v", err) + } + _, err = veleroV1Client.VolumeSnapshotLocations(velerov1.DefaultNamespace).Get(context.Background(), "default", metav1.GetOptions{}) + if err == nil { + count++ + } else if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Failed to get volume snapshot location: %v", err) + } + deployments, err := clientset.AppsV1().Deployments(velerov1.DefaultNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Errorf("Failed to list deployments: %v", err) + } + count += len(deployments.Items) + for _, deployment := range deployments.Items { + fmt.Printf("count deployment: %s\n", deployment.Name) + } + daemonsets, err := clientset.AppsV1().DaemonSets(velerov1.DefaultNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Errorf("Failed to list daemonsets: %v", err) + } + count += len(daemonsets.Items) + for _, daemonset := range daemonsets.Items { + fmt.Printf("count daemonset: %s\n", daemonset.Name) + } + // count the number of resources created + return true, count +} + +func kubeConfigBytesToTmpFile(t *testing.T, kubeconfigBytes []byte) string { + tmpFile, err := os.CreateTemp("", "kubeconfig") + if err != nil { + t.Errorf("Failed to create temp file: %v", err) + } + defer tmpFile.Close() + if _, err := tmpFile.Write(kubeconfigBytes); err != nil { + t.Errorf("Failed to write kubeconfig to temp file: %v", err) + } + return tmpFile.Name() +} + +/** + * Snippets from controller-runtime@v0.12.2/pkg/internal/testing/controlplane/kubectl.go + */ +const ( + envtestName = "envtest" +) + +// KubeConfigFromREST reverse-engineers a kubeconfig file from a rest.Config. +// The options are tailored towards the rest.Configs we generate, so they're +// not broadly applicable. +// +// This is not intended to be exposed beyond internal for the above reasons. +func KubeConfigFromREST(cfg *rest.Config) ([]byte, error) { + kubeConfig := kcapi.NewConfig() + protocol := "https" + if !rest.IsConfigTransportTLS(*cfg) { + protocol = "http" + } + + // cfg.Host is a URL, so we need to parse it so we can properly append the API path + baseURL, err := url.Parse(cfg.Host) + if err != nil { + return nil, fmt.Errorf("unable to interpret config's host value as a URL: %w", err) + } + + kubeConfig.Clusters[envtestName] = &kcapi.Cluster{ + // TODO(directxman12): if client-go ever decides to expose defaultServerUrlFor(config), + // we can just use that. Note that this is not the same as the public DefaultServerURL, + // which requires us to pass a bunch of stuff in manually. + Server: (&url.URL{Scheme: protocol, Host: baseURL.Host, Path: cfg.APIPath}).String(), + CertificateAuthorityData: cfg.CAData, + } + kubeConfig.AuthInfos[envtestName] = &kcapi.AuthInfo{ + // try to cover all auth strategies that aren't plugins + ClientCertificateData: cfg.CertData, + ClientKeyData: cfg.KeyData, + Token: cfg.BearerToken, + Username: cfg.Username, + Password: cfg.Password, + } + kcCtx := kcapi.NewContext() + kcCtx.Cluster = envtestName + kcCtx.AuthInfo = envtestName + kubeConfig.Contexts[envtestName] = kcCtx + kubeConfig.CurrentContext = envtestName + + contents, err := clientcmd.Write(*kubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to serialize kubeconfig file: %w", err) + } + return contents, nil +} diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index 884ba0312a..43f6468897 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -64,6 +64,8 @@ func NewCommand(f client.Factory) *cobra.Command { The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). Use '--force' to skip the prompt confirming if you want to uninstall Velero. + +If you need to preserve namespace, use velero install command with original install options and add '--uninstall' and '--preserve-uninstall-namespace' flags. `, Example: ` # velero uninstall --namespace staging`, Run: func(c *cobra.Command, args []string) { @@ -73,7 +75,7 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero. // Confirm if not asked to force-skip confirmation if !o.force { - fmt.Println("You are about to uninstall Velero.") + fmt.Printf("You are about to uninstall Velero from namespace %q. Namespace will be deleted\nTo uninstall from a different namespace, use --namespace flag.", f.Namespace()) if !cli.GetConfirmation() { // Don't do anything unless we get confirmation return diff --git a/pkg/install/install.go b/pkg/install/install.go index d378716ed5..8d871193da 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -297,6 +297,29 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory, return nil } +func deleteResource(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer) error { + id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName()) + + // Helper to reduce boilerplate message about the same object + log := func(f string, a ...interface{}) { + format := strings.Join([]string{id, ": ", f, "\n"}, "") + fmt.Fprintf(w, format, a...) + } + log("attempting to delete resource") + + c, err := CreateClient(r, factory, w) + if err != nil { + return err + } + + if err := c.Delete(r.GetName(), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "Error deleting resource %s", id) + } + + log("deleted") + return nil +} + // CreateClient creates a client for an unstructured resource func CreateClient(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer) (client.Dynamic, error) { id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName()) @@ -356,3 +379,24 @@ func Install(dynamicFactory client.DynamicFactory, kbClient kbclient.Client, res return nil } + +func Uninstall(dynamicFactory client.DynamicFactory, resources *unstructured.UnstructuredList, w io.Writer, preserveNamespace bool) error { + rg := GroupResources(resources) + + for _, r := range rg.OtherResources { + if r.GroupVersionKind().Kind == "Namespace" && preserveNamespace { + continue + } + if err := deleteResource(r, dynamicFactory, w); err != nil { + return err + } + } + + for _, r := range rg.CRDResources { + if err := deleteResource(r, dynamicFactory, w); err != nil { + return err + } + } + + return nil +}