From 40a094f0cc08dea349020776f0fbc0425bd6ce58 Mon Sep 17 00:00:00 2001
From: Mikalai Radchuk <mradchuk@redhat.com>
Date: Mon, 9 Oct 2023 16:35:46 +0100
Subject: [PATCH] Update E2E

* Add tests for semver upgrades
* Label semver and legacy tests so it is possible to filter one or the other out
* Update Makefile to no run legacy tests by default to match default deployment of operator-controller

Signed-off-by: Mikalai Radchuk <mradchuk@redhat.com>
---
 Makefile                                    |  14 ++-
 test/e2e/install_test.go                    | 124 +++++++++++++-------
 testdata/catalogs/test-catalog/catalog.yaml |  48 ++++++++
 3 files changed, 142 insertions(+), 44 deletions(-)

diff --git a/Makefile b/Makefile
index cad1265a6..d74ae1ee8 100644
--- a/Makefile
+++ b/Makefile
@@ -20,8 +20,8 @@ export XDG_DATA_HOME ?= /tmp/.local/share
 include .bingo/Variables.mk
 
 # ARTIFACT_PATH is the absolute path to the directory where the operator-controller e2e tests will store the artifacts
-# for example: ARTIFACT_PATH=/tmp/artifacts make test 
-export ARTIFACT_PATH ?= 
+# for example: ARTIFACT_PATH=/tmp/artifacts make test
+export ARTIFACT_PATH ?=
 
 OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system
 KIND_CLUSTER_NAME ?= operator-controller
@@ -104,7 +104,7 @@ test: manifests generate fmt vet test-unit test-e2e #HELP Run all tests.
 
 .PHONY: e2e
 FOCUS := $(if $(TEST),-v -focus "$(TEST)")
-E2E_FLAGS ?= ""
+E2E_FLAGS ?= --label-filter="!ForceSemverUpgradeConstraints=false"
 e2e: $(GINKGO) #EXHELP Run the e2e tests.
 	$(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -progress $(FOCUS) test/e2e
 
@@ -156,11 +156,19 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.37.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e
 	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME)
 	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME)
 	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0 --name $(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1 --name $(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME)
 	$(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME)
 	$(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME)
 
diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go
index ee1c3543f..3d2e42394 100644
--- a/test/e2e/install_test.go
+++ b/test/e2e/install_test.go
@@ -67,7 +67,7 @@ var _ = Describe("Operator Install", func() {
 					g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
 					g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
 					g.Expect(cond.Message).To(ContainSubstring("resolved to"))
-					g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1"))
+					g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0"))
 				}).Should(Succeed())
 
 				By("eventually installing the package successfully")
@@ -174,48 +174,90 @@ var _ = Describe("Operator Install", func() {
 			}).Should(Succeed())
 		})
 
-		It("handles upgrade edges correctly", func() {
-			By("creating a valid Operator resource")
-			operator.Spec = operatorv1alpha1.OperatorSpec{
-				PackageName: "prometheus",
-				Version:     "0.37.0",
-			}
-			Expect(c.Create(ctx, operator)).To(Succeed())
-			By("eventually reporting a successful resolution")
-			Eventually(func(g Gomega) {
-				g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
-				cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
-				g.Expect(cond).ToNot(BeNil())
-				g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
-				g.Expect(cond.Message).To(ContainSubstring("resolved to"))
-				g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty())
-			}).Should(Succeed())
+		When("resolving upgrade edges", func() {
+			BeforeEach(func() {
+				By("creating an Operator at a specified version")
+				operator.Spec = operatorv1alpha1.OperatorSpec{
+					PackageName: "prometheus",
+					Version:     "1.0.0",
+				}
+				Expect(c.Create(ctx, operator)).To(Succeed())
+				By("eventually reporting a successful resolution")
+				Eventually(func(g Gomega) {
+					g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
+					cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
+					g.Expect(cond).ToNot(BeNil())
+					g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
+					g.Expect(cond.Message).To(ContainSubstring("resolved to"))
+					g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0"))
+				}).Should(Succeed())
+			})
 
-			By("updating the Operator resource to a non-successor version")
-			operator.Spec.Version = "0.65.1" // current (0.37.0) and successor (0.47.0) are the only values that would be SAT.
-			Expect(c.Update(ctx, operator)).To(Succeed())
-			By("eventually reporting an unsatisfiable resolution")
-			Eventually(func(g Gomega) {
-				g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
-				cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
-				g.Expect(cond).ToNot(BeNil())
-				g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
-				g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*0.47.0[^,]*,[^,]*0.37.0[^;]*;.*`))
-				g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
-			}).Should(Succeed())
+			When("using legacy upgrade edge constraints from OLMv0", Label("ForceSemverUpgradeConstraints=false"), func() {
+				It("does not allow to upgrade the Operator to a non-successor version", func() {
+					By("updating the Operator resource to a non-successor version")
+					operator.Spec.Version = "1.2.0" // current (1.0.0) and successor (1.0.1) are the only values that would be SAT.
+					Expect(c.Update(ctx, operator)).To(Succeed())
+					By("eventually reporting an unsatisfiable resolution")
+					Eventually(func(g Gomega) {
+						g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
+						cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
+						g.Expect(cond).ToNot(BeNil())
+						g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
+						g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*1.0.1[^,]*,[^,]*1.0.0[^;]*;.*`))
+						g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
+					}).Should(Succeed())
+				})
+
+				It("does allow to upgrade the Operator to a successor version", func() {
+					By("updating the Operator resource to a valid upgrade edge")
+					operator.Spec.Version = "1.0.1"
+					Expect(c.Update(ctx, operator)).To(Succeed())
+					By("eventually reporting a successful resolution and bundle path")
+					Eventually(func(g Gomega) {
+						g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
+						cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
+						g.Expect(cond).ToNot(BeNil())
+						g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
+						g.Expect(cond.Message).To(ContainSubstring("resolved to"))
+						g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1"))
+					}).Should(Succeed())
+				})
+			})
 
-			By("updating the Operator resource to a valid upgrade edge")
-			operator.Spec.Version = "0.47.0"
-			Expect(c.Update(ctx, operator)).To(Succeed())
-			By("eventually reporting a successful resolution and bundle path")
-			Eventually(func(g Gomega) {
-				g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
-				cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
-				g.Expect(cond).ToNot(BeNil())
-				g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
-				g.Expect(cond.Message).To(ContainSubstring("resolved to"))
-				g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty())
-			}).Should(Succeed())
+			When("using semver upgrade edge constraints", Label("ForceSemverUpgradeConstraints=true"), func() {
+				It("does not allow to upgrade the Operator to a non-successor version", func() {
+					By("updating the Operator resource to a non-successor version")
+					// Semver only allows upgrades within major version at the moment.
+					operator.Spec.Version = "2.0.0"
+					Expect(c.Update(ctx, operator)).To(Succeed())
+					By("eventually reporting an unsatisfiable resolution")
+					Eventually(func(g Gomega) {
+						g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
+						cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
+						g.Expect(cond).ToNot(BeNil())
+						g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
+						g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*1.2.0[^;]*;.*`))
+						g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
+					}).Should(Succeed())
+				})
+
+				It("does allow to upgrade the Operator to any of the successor versions within non-zero major version", func() {
+					By("updating the Operator resource by skipping versions")
+					// Test catalog has versions between the initial version and new version
+					operator.Spec.Version = "1.2.0"
+					Expect(c.Update(ctx, operator)).To(Succeed())
+					By("eventually reporting a successful resolution and bundle path")
+					Eventually(func(g Gomega) {
+						g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
+						cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
+						g.Expect(cond).ToNot(BeNil())
+						g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
+						g.Expect(cond.Message).To(ContainSubstring("resolved to"))
+						g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0"))
+					}).Should(Succeed())
+				})
+			})
 		})
 
 		AfterEach(func() {
diff --git a/testdata/catalogs/test-catalog/catalog.yaml b/testdata/catalogs/test-catalog/catalog.yaml
index c82489b03..2109c7d3f 100644
--- a/testdata/catalogs/test-catalog/catalog.yaml
+++ b/testdata/catalogs/test-catalog/catalog.yaml
@@ -18,6 +18,14 @@ entries:
     replaces: prometheus-operator.0.37.0
   - name: prometheus-operator.0.65.1
     replaces: prometheus-operator.0.47.0
+  - name: prometheus-operator.1.0.0
+    replaces: prometheus-operator.0.65.1
+  - name: prometheus-operator.1.0.1
+    replaces: prometheus-operator.1.0.0
+  - name: prometheus-operator.1.2.0
+    replaces: prometheus-operator.1.0.1
+  - name: prometheus-operator.2.0.0
+    replaces: prometheus-operator.1.2.0
 ---
 schema: olm.bundle
 name: prometheus-operator.0.37.0
@@ -49,6 +57,46 @@ properties:
       packageName: prometheus
       version: 0.65.1
 ---
+schema: olm.bundle
+name: prometheus-operator.1.0.0
+package: prometheus
+image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0
+properties:
+  - type: olm.package
+    value:
+      packageName: prometheus
+      version: 1.0.0
+---
+schema: olm.bundle
+name: prometheus-operator.1.0.1
+package: prometheus
+image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1
+properties:
+  - type: olm.package
+    value:
+      packageName: prometheus
+      version: 1.0.1
+---
+schema: olm.bundle
+name: prometheus-operator.1.2.0
+package: prometheus
+image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
+properties:
+  - type: olm.package
+    value:
+      packageName: prometheus
+      version: 1.2.0
+---
+schema: olm.bundle
+name: prometheus-operator.2.0.0
+package: prometheus
+image: localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
+properties:
+  - type: olm.package
+    value:
+      packageName: prometheus
+      version: 2.0.0
+---
 schema: olm.package
 name: plain
 defaultChannel: beta