From fec525b7e20b137ea3f646a61f23671ba2abbb02 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 5 Dec 2023 15:44:19 +0100 Subject: [PATCH] clusterctl: validate no objects exist from CRDs before deleting them --- cmd/clusterctl/client/cluster/components.go | 58 ++++++++++++++ .../client/cluster/components_test.go | 76 +++++++++++++++++++ cmd/clusterctl/client/delete.go | 14 ++++ cmd/clusterctl/client/delete_test.go | 21 +++++ 4 files changed, 169 insertions(+) diff --git a/cmd/clusterctl/client/cluster/components.go b/cmd/clusterctl/client/cluster/components.go index 3bc6a00e05fe..6dcbfc35cb64 100644 --- a/cmd/clusterctl/client/cluster/components.go +++ b/cmd/clusterctl/client/cluster/components.go @@ -23,9 +23,11 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -66,6 +68,9 @@ type ComponentsClient interface { // DeleteWebhookNamespace deletes the core provider webhook namespace (eg. capi-webhook-system). // This is required when upgrading to v1alpha4 where webhooks are included in the controller itself. DeleteWebhookNamespace(ctx context.Context) error + + // ValidateNoObjectsExist checks if custom resources of the custom resource definitions exist and returns an error if so. + ValidateNoObjectsExist(ctx context.Context, provider clusterctlv1.Provider) error } // providerComponents implements ComponentsClient. @@ -257,6 +262,59 @@ func (p *providerComponents) DeleteWebhookNamespace(ctx context.Context) error { return nil } +func (p *providerComponents) ValidateNoObjectsExist(ctx context.Context, provider clusterctlv1.Provider) error { + log := logf.Log + log.Info("Checking for CRs", "Provider", provider.Name, "Version", provider.Version, "Namespace", provider.Namespace) + + proxyClient, err := p.proxy.NewClient() + if err != nil { + return err + } + + // Fetch all the components belonging to a provider. + // We want that the delete operation is able to clean-up everything. + labels := map[string]string{ + clusterctlv1.ClusterctlLabel: "", + clusterv1.ProviderNameLabel: provider.ManifestLabel(), + } + + customResources := &apiextensionsv1.CustomResourceDefinitionList{} + if err := proxyClient.List(ctx, customResources, client.MatchingLabels(labels)); err != nil { + return err + } + + // Filter the resources according to the delete options + crsHavingObjects := []string{} + for _, crd := range customResources.Items { + crd := crd + storageVersion, err := storageVersionForCRD(&crd) + if err != nil { + return err + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: storageVersion, + Kind: crd.Spec.Names.ListKind, + }) + + if err := proxyClient.List(ctx, list); err != nil { + return err + } + + if len(list.Items) > 0 { + crsHavingObjects = append(crsHavingObjects, crd.Kind) + } + } + + if len(crsHavingObjects) > 0 { + return fmt.Errorf("found existing objects for provider CRDs %q: [%s]. Please delete these objects first before running clusterctl delete with --include-crd", provider.GetName(), strings.Join(crsHavingObjects, ", ")) + } + + return nil +} + // newComponentsClient returns a providerComponents. func newComponentsClient(proxy Proxy) *providerComponents { return &providerComponents{ diff --git a/cmd/clusterctl/client/cluster/components_test.go b/cmd/clusterctl/client/cluster/components_test.go index d2affd1ddaa5..cbcd7da43d89 100644 --- a/cmd/clusterctl/client/cluster/components_test.go +++ b/cmd/clusterctl/client/cluster/components_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -495,3 +496,78 @@ func Test_providerComponents_Create(t *testing.T) { }) } } + +func Test_providerComponents_ValidateNoObjectsExist(t *testing.T) { + labels := map[string]string{ + clusterv1.ProviderNameLabel: "infrastructure-infra", + } + + crd := &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: "CustomResourceDefinition", + APIVersion: apiextensionsv1.SchemeGroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "crd1", + Labels: labels, + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "some.group", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + ListKind: "SomeCRDList", + Kind: "SomeCRD", + }, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + {Name: "v1", Storage: true}, + }, + }, + } + crd.ObjectMeta.Labels[clusterctlv1.ClusterctlLabel] = "" + + cr := &unstructured.Unstructured{} + cr.SetAPIVersion("some.group/v1") + cr.SetKind("SomeCRD") + cr.SetName("cr1") + + tests := []struct { + name string + provider clusterctlv1.Provider + initObjs []client.Object + wantErr bool + }{ + { + name: "No objects exist", + provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)}, + initObjs: []client.Object{}, + wantErr: false, + }, + { + name: "CRD exists but no objects", + provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)}, + initObjs: []client.Object{ + crd, + }, + wantErr: false, + }, + { + name: "CRD exists but and also objects", + provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)}, + initObjs: []client.Object{ + crd, + cr, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy := test.NewFakeProxy().WithObjs(tt.initObjs...) + + c := newComponentsClient(proxy) + + if err := c.ValidateNoObjectsExist(context.Background(), tt.provider); (err != nil) != tt.wantErr { + t.Errorf("providerComponents.ValidateNoObjectsExist() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/clusterctl/client/delete.go b/cmd/clusterctl/client/delete.go index f59f443544a6..68b124b48668 100644 --- a/cmd/clusterctl/client/delete.go +++ b/cmd/clusterctl/client/delete.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" @@ -156,6 +157,19 @@ func (c *clusterctlClient) Delete(ctx context.Context, options DeleteOptions) er } } + if options.IncludeCRDs { + errList := []error{} + for _, provider := range providersToDelete { + err = clusterClient.ProviderComponents().ValidateNoObjectsExist(ctx, provider) + if err != nil { + errList = append(errList, err) + } + } + if len(errList) > 0 { + return kerrors.NewAggregate(errList) + } + } + // Delete the selected providers. for _, provider := range providersToDelete { if err := clusterClient.ProviderComponents().Delete(ctx, cluster.DeleteOptions{Provider: provider, IncludeNamespace: options.IncludeNamespace, IncludeCRDs: options.IncludeCRDs, SkipInventory: options.SkipInventory}); err != nil { diff --git a/cmd/clusterctl/client/delete_test.go b/cmd/clusterctl/client/delete_test.go index 52e9d4b03970..cfdfa870cabd 100644 --- a/cmd/clusterctl/client/delete_test.go +++ b/cmd/clusterctl/client/delete_test.go @@ -67,6 +67,27 @@ func Test_clusterctlClient_Delete(t *testing.T) { wantProviders: sets.Set[string]{}, wantErr: false, }, + { + name: "Delete all the providers including CRDs", + fields: fields{ + client: fakeClusterForDelete(), + }, + args: args{ + options: DeleteOptions{ + Kubeconfig: Kubeconfig{Path: "kubeconfig", Context: "mgmt-context"}, + IncludeNamespace: false, + IncludeCRDs: true, + SkipInventory: false, + CoreProvider: "", + BootstrapProviders: nil, + InfrastructureProviders: nil, + ControlPlaneProviders: nil, + DeleteAll: true, // delete all the providers + }, + }, + wantProviders: sets.Set[string]{}, + wantErr: false, + }, { name: "Delete single provider auto-detect namespace", fields: fields{