Merge pull request #697 from pires/pires/delete_crds_on_stop

✨ Clean-up installed CRDs on Stop()

pkg/envtest/crd.go

@@ -27,6 +27,8 @@ import (
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -51,6 +53,11 @@ type CRDInstallOptions struct {
 
 	// PollInterval is the interval to check
 	PollInterval time.Duration
+
+	// CleanUpAfterUse will cause the CRDs listed for installation to be
+	// uninstalled when terminating the test environment.
+	// Defaults to false.
+	CleanUpAfterUse bool
 }
 
 const defaultPollInterval = 100 * time.Millisecond
@@ -180,6 +187,34 @@ func (p *poller) poll() (done bool, err error) {
 	return allFound, nil
 }
 
+// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory
+func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
+
+	// Read the CRD yamls into options.CRDs
+	if err := readCRDFiles(&options); err != nil {
+		return err
+	}
+
+	// Delete the CRDs from the apiserver
+	cs, err := clientset.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	// Uninstall each CRD
+	for _, crd := range options.CRDs {
+		log.V(1).Info("uninstalling CRD", "crd", crd.Name)
+		if err := cs.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(crd.Name, &metav1.DeleteOptions{}); err != nil {
+			// If CRD is not found, we can consider success
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // CreateCRDs creates the CRDs
 func CreateCRDs(config *rest.Config, crds []*apiextensionsv1beta1.CustomResourceDefinition) error {
 	cs, err := clientset.NewForConfig(config)

pkg/envtest/envtest_test.go

@@ -19,12 +19,12 @@ package envtest
 import (
 	"context"
 	"path/filepath"
-
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -52,6 +52,17 @@ var _ = Describe("Test", func() {
 	// Cleanup CRDs
 	AfterEach(func(done Done) {
 		for _, crd := range crds {
+			// Delete only if CRD exists.
+			crdObjectKey := client.ObjectKey{
+				Name: crd.Name,
+			}
+			var placeholder v1beta1.CustomResourceDefinition
+			err := c.Get(context.TODO(), crdObjectKey, &placeholder)
+			if err != nil && apierrors.IsNotFound(err) {
+				// CRD doesn't need to be deleted.
+				continue
+			}
+			Expect(err).NotTo(HaveOccurred())
 			Expect(c.Delete(context.TODO(), crd)).To(Succeed())
 		}
 		close(done)
@@ -216,4 +227,126 @@ var _ = Describe("Test", func() {
 			close(done)
 		}, 5)
 	})
+
+	Describe("UninstallCRDs", func() {
+		It("should uninstall the CRDs from the cluster", func(done Done) {
+
+			crds, err = InstallCRDs(env.Config, CRDInstallOptions{
+				Paths: []string{filepath.Join(".", "testdata")},
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Expect to find the CRDs
+
+			crd := &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Foo"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Baz"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Captain"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("FirstMate"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Driver"))
+
+			err = WaitForCRDs(env.Config, []*v1beta1.CustomResourceDefinition{
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "qux.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "bazs",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "bar.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "foos",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "crew.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "captains",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "crew.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "firstmates",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group: "crew.example.com",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "drivers",
+						},
+						Versions: []v1beta1.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Storage: true,
+								Served:  true,
+							},
+							{
+								Name:    "v2",
+								Storage: false,
+								Served:  true,
+							},
+						}},
+				},
+			},
+				CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond},
+			)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = UninstallCRDs(env.Config, CRDInstallOptions{
+				Paths: []string{filepath.Join(".", "testdata")},
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Expect to NOT find the CRDs
+
+			crds := []string{
+				"foos.bar.example.com",
+				"bazs.qux.example.com",
+				"captains.crew.example.com",
+				"firstmates.crew.example.com",
+				"drivers.crew.example.com",
+			}
+			placeholder := &v1beta1.CustomResourceDefinition{}
+			Eventually(func() bool {
+				for _, crd := range crds {
+					err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, placeholder)
+					notFound := err != nil && apierrors.IsNotFound(err)
+					if !notFound {
+						return false
+					}
+				}
+				return true
+			}, 20).Should(BeTrue())
+
+			close(done)
+		}, 30)
+	})
 })

pkg/envtest/server.go

@@ -135,8 +135,14 @@ type Environment struct {
 }
 
 // Stop stops a running server.
-// If USE_EXISTING_CLUSTER is set to true, this method is a no-op.
+// Previously installed CRDs, as listed in CRDInstallOptions.CRDs, will be uninstalled
+// if CRDInstallOptions.CleanUpAfterUse are set to true.
 func (te *Environment) Stop() error {
+	if te.CRDInstallOptions.CleanUpAfterUse {
+		if err := UninstallCRDs(te.Config, te.CRDInstallOptions); err != nil {
+			return err
+		}
+	}
 	if te.useExistingCluster() {
 		return nil
 	}