Skip to content

Commit

Permalink
Update E2E
Browse files Browse the repository at this point in the history 
* Add tests for semver upgrades
* Label semver and legacy tests so it is possible to filter one or the other out
* Update Makefile to no run legacy tests by default to match default deployment of operator-controller

Signed-off-by: Mikalai Radchuk <mradchuk@redhat.com>
  • Loading branch information
@m1kola
m1kola committed Oct 11, 2023
1 parent f316ee4 commit b0c17cb
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 44 deletions.
14 changes: 11 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ export XDG_DATA_HOME ?= /tmp/.local/share
include .bingo/Variables.mk

# ARTIFACT_PATH is the absolute path to the directory where the operator-controller e2e tests will store the artifacts
# for example: ARTIFACT_PATH=/tmp/artifacts make test
export ARTIFACT_PATH ?=
# for example: ARTIFACT_PATH=/tmp/artifacts make test
export ARTIFACT_PATH ?=

OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system
KIND_CLUSTER_NAME ?= operator-controller
Expand Down Expand Up @@ -104,7 +104,7 @@ test: manifests generate fmt vet test-unit test-e2e #HELP Run all tests.

.PHONY: e2e
FOCUS := $(if $(TEST),-v -focus "$(TEST)")
E2E_FLAGS ?= ""
E2E_FLAGS ?= --label-filter="!ForceSemverUpgradeConstraints=false"
e2e: $(GINKGO) #EXHELP Run the e2e tests.
$(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -progress $(FOCUS) test/e2e

Expand Down Expand Up @@ -156,11 +156,19 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.37.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.65.1 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME)

Expand Down
124 changes: 83 additions & 41 deletions test/e2e/install_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ var _ = Describe("Operator Install", func() {
g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1"))
g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0"))
}).Should(Succeed())

By("eventually installing the package successfully")
Expand Down Expand Up @@ -174,48 +174,90 @@ var _ = Describe("Operator Install", func() {
}).Should(Succeed())
})

It("handles upgrade edges correctly", func() {
By("creating a valid Operator resource")
operator.Spec = operatorv1alpha1.OperatorSpec{
PackageName: "prometheus",
Version: "0.37.0",
}
Expect(c.Create(ctx, operator)).To(Succeed())
By("eventually reporting a successful resolution")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty())
}).Should(Succeed())
When("resolving upgrade edges", func() {
BeforeEach(func() {
By("creating an Operator at a specified version")
operator.Spec = operatorv1alpha1.OperatorSpec{
PackageName: "prometheus",
Version: "1.0.0",
}
Expect(c.Create(ctx, operator)).To(Succeed())
By("eventually reporting a successful resolution")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0"))
}).Should(Succeed())
})

By("updating the Operator resource to a non-successor version")
operator.Spec.Version = "0.65.1" // current (0.37.0) and successor (0.47.0) are the only values that would be SAT.
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting an unsatisfiable resolution")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*0.47.0[^,]*,[^,]*0.37.0[^;]*;.*`))
g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
}).Should(Succeed())
When("using legacy upgrade edge constraints from OLMv0", Label("ForceSemverUpgradeConstraints=false"), func() {
It("does not allow to upgrade the Operator to a non-successor version", func() {
By("updating the Operator resource to a non-successor version")
operator.Spec.Version = "1.2.0" // current (1.0.0) and successor (1.0.1) are the only values that would be SAT.
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting an unsatisfiable resolution")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*1.0.1[^,]*,[^,]*1.0.0[^;]*;.*`))
g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
}).Should(Succeed())
})

It("does allow to upgrade the Operator to a successor version", func() {
By("updating the Operator resource to a valid upgrade edge")
operator.Spec.Version = "1.0.1"
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting a successful resolution and bundle path")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1"))
}).Should(Succeed())
})
})

By("updating the Operator resource to a valid upgrade edge")
operator.Spec.Version = "0.47.0"
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting a successful resolution and bundle path")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).ToNot(BeEmpty())
}).Should(Succeed())
When("using semver upgrade edge constraints", Label("ForceSemverUpgradeConstraints=true"), func() {
It("does not allow to upgrade the Operator to a non-successor version", func() {
By("updating the Operator resource to a non-successor version")
// Semver only allows upgrades within major version at the moment.
operator.Spec.Version = "2.0.0"
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting an unsatisfiable resolution")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonResolutionFailed))
g.Expect(cond.Message).To(MatchRegexp(`^constraints not satisfiable:.*; installed package prometheus requires at least one of.*1.2.0[^;]*;.*`))
g.Expect(operator.Status.ResolvedBundleResource).To(BeEmpty())
}).Should(Succeed())
})

It("does allow to upgrade the Operator to any of the successor versions within non-zero major version", func() {
By("updating the Operator resource by skipping versions")
// Test catalog has versions between the initial version and new version
operator.Spec.Version = "1.2.0"
Expect(c.Update(ctx, operator)).To(Succeed())
By("eventually reporting a successful resolution and bundle path")
Eventually(func(g Gomega) {
g.Expect(c.Get(ctx, types.NamespacedName{Name: operator.Name}, operator)).To(Succeed())
cond := apimeta.FindStatusCondition(operator.Status.Conditions, operatorv1alpha1.TypeResolved)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Reason).To(Equal(operatorv1alpha1.ReasonSuccess))
g.Expect(cond.Message).To(ContainSubstring("resolved to"))
g.Expect(operator.Status.ResolvedBundleResource).To(Equal("localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0"))
}).Should(Succeed())
})
})
})

AfterEach(func() {
Expand Down
48 changes: 48 additions & 0 deletions testdata/catalogs/test-catalog/catalog.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ entries:
replaces: prometheus-operator.0.37.0
- name: prometheus-operator.0.65.1
replaces: prometheus-operator.0.47.0
- name: prometheus-operator.1.0.0
replaces: prometheus-operator.0.65.1
- name: prometheus-operator.1.0.1
replaces: prometheus-operator.1.0.0
- name: prometheus-operator.1.2.0
replaces: prometheus-operator.1.0.1
- name: prometheus-operator.2.0.0
replaces: prometheus-operator.1.2.0
---
schema: olm.bundle
name: prometheus-operator.0.37.0
Expand Down Expand Up @@ -49,6 +57,46 @@ properties:
packageName: prometheus
version: 0.65.1
---
schema: olm.bundle
name: prometheus-operator.1.0.0
package: prometheus
image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.0
properties:
- type: olm.package
value:
packageName: prometheus
version: 1.0.0
---
schema: olm.bundle
name: prometheus-operator.1.0.1
package: prometheus
image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.0.1
properties:
- type: olm.package
value:
packageName: prometheus
version: 1.0.1
---
schema: olm.bundle
name: prometheus-operator.1.2.0
package: prometheus
image: localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
properties:
- type: olm.package
value:
packageName: prometheus
version: 1.2.0
---
schema: olm.bundle
name: prometheus-operator.2.0.0
package: prometheus
image: localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
properties:
- type: olm.package
value:
packageName: prometheus
version: 2.0.0
---
schema: olm.package
name: plain
defaultChannel: beta
Expand Down

0 comments on commit b0c17cb

Please sign in to comment.