Add upgrade E2E (#1003)

Testing upgrade from the latest release to the current commit

Signed-off-by: Mikalai Radchuk <mradchuk@redhat.com>

.github/workflows/e2e.yaml

@@ -19,8 +19,7 @@ jobs:
         go-version-file: go.mod
 
     - name: Run the extension developer e2e test
-      run: |
-        make extension-developer-e2e
+      run: make extension-developer-e2e
 
   e2e-kind:
     runs-on: ubuntu-latest
@@ -48,3 +47,15 @@ jobs:
           files: e2e-cover.out
           flags: e2e
           token: ${{ secrets.CODECOV_TOKEN }}
+
+  upgrade-e2e:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Run the upgrade e2e test
+      run: make test-upgrade-e2e

Makefile

@@ -159,6 +159,24 @@ extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager
 extension-developer-e2e: KIND_CLUSTER_NAME := operator-controller-ext-dev-e2e  #EXHELP Run extension-developer e2e on local kind cluster
 extension-developer-e2e: run image-registry test-ext-dev-e2e kind-clean
 
+.PHONY: run-latest-release
+run-latest-release:
+	curl -L -s https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh | bash -s
+
+.PHONY: pre-upgrade-setup
+pre-upgrade-setup:
+	./hack/pre-upgrade-setup.sh $(CATALOG_IMG) $(TEST_CLUSTER_CATALOG_NAME) $(TEST_CLUSTER_EXTENSION_NAME)
+
+.PHONY: post-upgrade-checks
+post-upgrade-checks:
+	go test -count=1 -v ./test/upgrade-e2e/...
+
+.PHONY: test-upgrade-e2e
+test-upgrade-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-e2e
+test-upgrade-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
+test-upgrade-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package
+test-upgrade-e2e: kind-cluster run-latest-release image-registry build-push-e2e-catalog registry-load-bundles pre-upgrade-setup docker-build kind-load kind-deploy post-upgrade-checks kind-clean #HELP Run upgrade e2e tests on a local kind cluster
+
 .PHONY: e2e-coverage
 e2e-coverage:
 	COVERAGE_OUTPUT=./e2e-cover.out ./hack/e2e-coverage.sh

hack/pre-upgrade-setup.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+
+help="pre-upgrade-setup.sh is used to create some basic resources
+which will later be used in upgrade testing.
+
+Usage:
+  post-upgrade-checks.sh [TEST_CATALOG_IMG] [TEST_CATALOG_NAME] [TEST_CLUSTER_EXTENSION_NAME]
+"
+
+if [[ "$#" -ne 3 ]]; then
+  echo "Illegal number of arguments passed"
+  echo "${help}"
+  exit 1
+fi
+
+TEST_CATALOG_IMG=$1
+TEST_CLUSTER_CATALOG_NAME=$2
+TEST_CLUSTER_EXTENSION_NAME=$3
+
+kubectl apply -f - << EOF
+apiVersion: catalogd.operatorframework.io/v1alpha1
+kind: ClusterCatalog
+metadata:
+  name: ${TEST_CLUSTER_CATALOG_NAME}
+spec:
+  source:
+    type: image
+    image:
+      ref: ${TEST_CATALOG_IMG}
+      pollInterval: 24h
+      insecureSkipTLSVerify: true
+EOF
+
+
+kubectl apply -f - << EOF
+apiVersion: olm.operatorframework.io/v1alpha1
+kind: ClusterExtension
+metadata:
+  name: ${TEST_CLUSTER_EXTENSION_NAME}
+spec:
+  installNamespace: default
+  packageName: prometheus
+  version: 1.0.0
+  serviceAccount:
+    name: default
+EOF
+
+kubectl wait --for=condition=Unpacked --timeout=60s ClusterCatalog $TEST_CLUSTER_CATALOG_NAME
+kubectl wait --for=condition=Installed --timeout=60s ClusterExtension $TEST_CLUSTER_EXTENSION_NAME

test/upgrade-e2e/post_upgrade_test.go

@@ -0,0 +1,135 @@
+package upgradee2e
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1"
+
+	ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1"
+)
+
+func TestClusterExtensionAfterOLMUpgrade(t *testing.T) {
+	t.Log("Starting checks after OLM upgrade")
+	ctx := context.Background()
+
+	managerLabelSelector := labels.Set{"control-plane": "controller-manager"}
+
+	t.Log("Checking that the controller-manager deployment is updated")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var managerDeployments appsv1.DeploymentList
+		assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()}))
+		assert.Len(ct, managerDeployments.Items, 1)
+		managerDeployment := managerDeployments.Items[0]
+
+		assert.True(ct,
+			managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas &&
+				managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas &&
+				managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas &&
+				managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas,
+		)
+	}, time.Minute, time.Second)
+
+	var managerPods corev1.PodList
+	t.Log("Waiting for only one controller-manager Pod to remain")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()}))
+		assert.Len(ct, managerPods.Items, 1)
+	}, time.Minute, time.Second)
+
+	t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it")
+	// Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes
+	logCtx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+	substring := fmt.Sprintf(`"ClusterExtension": {"name":"%s"}`, testClusterExtensionName)
+	found, err := watchPodLogsForSubstring(logCtx, &managerPods.Items[0], "manager", substring)
+	require.NoError(t, err)
+	require.True(t, found)
+
+	t.Log("Checking that the ClusterCatalog is unpacked")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var clusterCatalog catalogdv1alpha1.ClusterCatalog
+		assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog))
+		cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogdv1alpha1.TypeUnpacked)
+		if !assert.NotNil(ct, cond) {
+			return
+		}
+		assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+		assert.Equal(ct, catalogdv1alpha1.ReasonUnpackSuccessful, cond.Reason)
+	}, time.Minute, time.Second)
+
+	t.Log("Checking that the ClusterExtension is installed")
+	var clusterExtension ocv1alpha1.ClusterExtension
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled)
+		if !assert.NotNil(ct, cond) {
+			return
+		}
+		assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+		assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason)
+		assert.Contains(ct, cond.Message, "Instantiated bundle")
+		assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle)
+		assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle.Version)
+	}, time.Minute, time.Second)
+
+	previousVersion := clusterExtension.Status.InstalledBundle.Version
+
+	t.Log("Updating the ClusterExtension to change version")
+	// Make sure that after we upgrade OLM itself we can still reconcile old objects if we change them
+	clusterExtension.Spec.Version = "1.0.1"
+	require.NoError(t, c.Update(ctx, &clusterExtension))
+
+	t.Log("Checking that the ClusterExtension installs successfully")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled)
+		if !assert.NotNil(ct, cond) {
+			return
+		}
+		assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason)
+		assert.Contains(ct, cond.Message, "Instantiated bundle")
+		assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.ResolvedBundle)
+		assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.InstalledBundle)
+		assert.NotEqual(ct, previousVersion, clusterExtension.Status.InstalledBundle.Version)
+	}, time.Minute, time.Second)
+}
+
+func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container, substring string) (bool, error) {
+	podLogOpts := corev1.PodLogOptions{
+		Follow:    true,
+		Container: container,
+	}
+
+	req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts)
+	podLogs, err := req.Stream(ctx)
+	if err != nil {
+		return false, err
+	}
+	defer podLogs.Close()
+
+	scanner := bufio.NewScanner(podLogs)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if strings.Contains(line, substring) {
+			return true, nil
+		}
+	}
+
+	return false, scanner.Err()
+}

test/upgrade-e2e/upgrade_e2e_suite_test.go

@@ -0,0 +1,57 @@
+package upgradee2e
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"k8s.io/client-go/kubernetes"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/operator-framework/operator-controller/pkg/scheme"
+)
+
+const (
+	testClusterCatalogNameEnv   = "TEST_CLUSTER_CATALOG_NAME"
+	testClusterExtensionNameEnv = "TEST_CLUSTER_EXTENSION_NAME"
+)
+
+var (
+	c          client.Client
+	kclientset kubernetes.Interface
+
+	testClusterCatalogName   string
+	testClusterExtensionName string
+)
+
+func TestMain(m *testing.M) {
+	var ok bool
+	testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv)
+	if !ok {
+		fmt.Printf("%q is not set", testClusterCatalogNameEnv)
+		os.Exit(1)
+	}
+	testClusterExtensionName, ok = os.LookupEnv(testClusterExtensionNameEnv)
+	if !ok {
+		fmt.Printf("%q is not set", testClusterExtensionNameEnv)
+		os.Exit(1)
+	}
+
+	cfg := ctrl.GetConfigOrDie()
+
+	var err error
+	c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	if err != nil {
+		fmt.Printf("failed to create client: %s\n", err)
+		os.Exit(1)
+	}
+
+	kclientset, err = kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Printf("failed to create kubernetes clientset: %s\n", err)
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}