From 1e8ed0acecb5251edcfa9c1760cb254ab664a25f Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk Date: Mon, 9 Oct 2023 16:35:46 +0100 Subject: [PATCH] Update E2E * Add tests for semver upgrades * Label semver and legacy tests so it is possible to filter one or the other out * Update Makefile to no run legacy tests by default to match default deployment of operator-controller Signed-off-by: Mikalai Radchuk --- Makefile | 12 ++- test/e2e/install_test.go | 90 +++++++++++---------- testdata/catalogs/test-catalog/catalog.yaml | 48 +++++++++++ 3 files changed, 107 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index cad1265a6..2964debff 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ export XDG_DATA_HOME ?= /tmp/.local/share include .bingo/Variables.mk # ARTIFACT_PATH is the absolute path to the directory where the operator-controller e2e tests will store the artifacts -# for example: ARTIFACT_PATH=/tmp/artifacts make test -export ARTIFACT_PATH ?= +# for example: ARTIFACT_PATH=/tmp/artifacts make test +export ARTIFACT_PATH ?= OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system KIND_CLUSTER_NAME ?= operator-controller @@ -156,11 +156,19 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.37.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 + $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0 + $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1 + $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 + $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME) + $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0 --name $(KIND_CLUSTER_NAME) + $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1 --name $(KIND_CLUSTER_NAME) + $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME) + $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME) diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go index 368970cdf..fb19b59a7 100644 --- a/test/e2e/install_test.go +++ b/test/e2e/install_test.go @@ -67,7 +67,7 @@ var _ = Describe("Operator Install", func() { g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess)) g.Expect(cond.Message).To(ContainSubstring("resolved to")) - g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1")) + g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0")) }).Should(Succeed()) By("eventually installing the package successfully") @@ -180,48 +180,56 @@ var _ = Describe("Operator Install", func() { }).Should(Succeed()) }) - It("handles upgrade edges correctly", func() { - By("creating a valid Operator resource") - operator.Spec = operatorv1alpha1.OperatorSpec{ - PackageName: "prometheus", - Version: "0.37.0", - } - Expect(c.Create(ctx, operator)).To(Succeed()) - By("eventually reporting a successful resolution") - Eventually(func(g Gomega) { - g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) - cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) - g.Expect(cond).ToNot(BeNil()) - g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess)) - g.Expect(cond.Message).To(ContainSubstring("resolved to")) - g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty()) - }).Should(Succeed()) + When("resolving upgrade edges", func() { + BeforeEach(func() { + By("creating an Operator at a specified version") + operator.Spec = operatorv1alpha1.OperatorSpec{ + PackageName: "prometheus", + Version: "1.0.0", + } + Expect(c.Create(ctx, operator)).To(Succeed()) + By("eventually reporting a successful resolution") + Eventually(func(g Gomega) { + g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) + cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess)) + g.Expect(cond.Message).To(ContainSubstring("resolved to")) + g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0")) + }).Should(Succeed()) + }) - By("updating the Operator resource to a non-successor version") - operator.Spec.Version = "0.65.1" // current (0.37.0) and successor (0.47.0) are the only values that would be SAT. - Expect(c.Update(ctx, operator)).To(Succeed()) - By("eventually reporting an unsatisfiable resolution") - Eventually(func(g Gomega) { - g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) - cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) - g.Expect(cond).ToNot(BeNil()) - g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed)) - g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*0.47.0[^,]*,[^,]*0.37.0[^;]*;.*`)) - g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty()) - }).Should(Succeed()) + It("does not allow to upgrade the Operator to a non-successor version", func() { + By("updating the Operator resource to a non-successor version") + // Semver only allows upgrades within major version at the moment. + operator.Spec.Version = "2.0.0" + Expect(c.Update(ctx, operator)).To(Succeed()) + By("eventually reporting an unsatisfiable resolution") + Eventually(func(g Gomega) { + g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) + cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed)) + g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*1.2.0[^;]*;.*`)) + g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty()) + }).Should(Succeed()) + }) - By("updating the Operator resource to a valid upgrade edge") - operator.Spec.Version = "0.47.0" - Expect(c.Update(ctx, operator)).To(Succeed()) - By("eventually reporting a successful resolution and bundle path") - Eventually(func(g Gomega) { - g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) - cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) - g.Expect(cond).ToNot(BeNil()) - g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess)) - g.Expect(cond.Message).To(ContainSubstring("resolved to")) - g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty()) - }).Should(Succeed()) + It("does allow to upgrade the Operator to any of the successor versions within non-zero major version", func() { + By("updating the Operator resource by skipping versions") + // Test catalog has versions between the initial version and new version + operator.Spec.Version = "1.2.0" + Expect(c.Update(ctx, operator)).To(Succeed()) + By("eventually reporting a successful resolution and bundle path") + Eventually(func(g Gomega) { + g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed()) + cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess)) + g.Expect(cond.Message).To(ContainSubstring("resolved to")) + g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0")) + }).Should(Succeed()) + }) }) AfterEach(func() { diff --git a/testdata/catalogs/test-catalog/catalog.yaml b/testdata/catalogs/test-catalog/catalog.yaml index c82489b03..2109c7d3f 100644 --- a/testdata/catalogs/test-catalog/catalog.yaml +++ b/testdata/catalogs/test-catalog/catalog.yaml @@ -18,6 +18,14 @@ entries: replaces: prometheus-operator.0.37.0 - name: prometheus-operator.0.65.1 replaces: prometheus-operator.0.47.0 + - name: prometheus-operator.1.0.0 + replaces: prometheus-operator.0.65.1 + - name: prometheus-operator.1.0.1 + replaces: prometheus-operator.1.0.0 + - name: prometheus-operator.1.2.0 + replaces: prometheus-operator.1.0.1 + - name: prometheus-operator.2.0.0 + replaces: prometheus-operator.1.2.0 --- schema: olm.bundle name: prometheus-operator.0.37.0 @@ -49,6 +57,46 @@ properties: packageName: prometheus version: 0.65.1 --- +schema: olm.bundle +name: prometheus-operator.1.0.0 +package: prometheus +image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0 +properties: + - type: olm.package + value: + packageName: prometheus + version: 1.0.0 +--- +schema: olm.bundle +name: prometheus-operator.1.0.1 +package: prometheus +image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1 +properties: + - type: olm.package + value: + packageName: prometheus + version: 1.0.1 +--- +schema: olm.bundle +name: prometheus-operator.1.2.0 +package: prometheus +image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 +properties: + - type: olm.package + value: + packageName: prometheus + version: 1.2.0 +--- +schema: olm.bundle +name: prometheus-operator.2.0.0 +package: prometheus +image: localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 +properties: + - type: olm.package + value: + packageName: prometheus + version: 2.0.0 +--- schema: olm.package name: plain defaultChannel: beta