From 2cf7d2e3892410aca66fbec59a336061b99696fc Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk Date: Fri, 5 Jul 2024 14:03:50 +0200 Subject: [PATCH] Add upgrade E2E Testing upgrade from the latest release to the current commit Signed-off-by: Mikalai Radchuk --- .github/workflows/e2e.yaml | 15 ++- Makefile | 18 +++ hack/pre-upgrade-setup.sh | 51 ++++++++ test/upgrade-e2e/post_upgrade_test.go | 135 +++++++++++++++++++++ test/upgrade-e2e/upgrade_e2e_suite_test.go | 57 +++++++++ 5 files changed, 274 insertions(+), 2 deletions(-) create mode 100755 hack/pre-upgrade-setup.sh create mode 100644 test/upgrade-e2e/post_upgrade_test.go create mode 100644 test/upgrade-e2e/upgrade_e2e_suite_test.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ca3b4645..58ca89ed 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,8 +19,7 @@ jobs: go-version-file: go.mod - name: Run the extension developer e2e test - run: | - make extension-developer-e2e + run: make extension-developer-e2e e2e-kind: runs-on: ubuntu-latest @@ -48,3 +47,15 @@ jobs: files: e2e-cover.out flags: e2e token: ${{ secrets.CODECOV_TOKEN }} + + upgrade-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run the upgrade e2e test + run: make test-upgrade-e2e diff --git a/Makefile b/Makefile index a7409fd8..b4c784a7 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,24 @@ extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager extension-developer-e2e: KIND_CLUSTER_NAME := operator-controller-ext-dev-e2e #EXHELP Run extension-developer e2e on local kind cluster extension-developer-e2e: run image-registry test-ext-dev-e2e kind-clean +.PHONY: run-latest-release +run-latest-release: + curl -L -s https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh | bash -s + +.PHONY: pre-upgrade-setup +pre-upgrade-setup: + ./hack/pre-upgrade-setup.sh $(CATALOG_IMG) $(TEST_CLUSTER_CATALOG_NAME) $(TEST_CLUSTER_EXTENSION_NAME) + +.PHONY: post-upgrade-checks +post-upgrade-checks: + go test -count=1 -v ./test/upgrade-e2e/... + +.PHONY: test-upgrade-e2e +test-upgrade-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-e2e +test-upgrade-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog +test-upgrade-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package +test-upgrade-e2e: kind-cluster run-latest-release image-registry build-push-e2e-catalog registry-load-bundles pre-upgrade-setup docker-build kind-load kind-deploy post-upgrade-checks kind-clean #HELP Run upgrade e2e tests on a local kind cluster + .PHONY: e2e-coverage e2e-coverage: COVERAGE_OUTPUT=./e2e-cover.out ./hack/e2e-coverage.sh diff --git a/hack/pre-upgrade-setup.sh b/hack/pre-upgrade-setup.sh new file mode 100755 index 00000000..937b3837 --- /dev/null +++ b/hack/pre-upgrade-setup.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euo pipefail + +help="pre-upgrade-setup.sh is used to create some basic resources +which will later be used in upgrade testing. + +Usage: + post-upgrade-checks.sh [TEST_CATALOG_IMG] [TEST_CATALOG_NAME] [TEST_CLUSTER_EXTENSION_NAME] +" + +if [[ "$#" -ne 3 ]]; then + echo "Illegal number of arguments passed" + echo "${help}" + exit 1 +fi + +TEST_CATALOG_IMG=$1 +TEST_CLUSTER_CATALOG_NAME=$2 +TEST_CLUSTER_EXTENSION_NAME=$3 + +kubectl apply -f - << EOF +apiVersion: catalogd.operatorframework.io/v1alpha1 +kind: ClusterCatalog +metadata: + name: ${TEST_CLUSTER_CATALOG_NAME} +spec: + source: + type: image + image: + ref: ${TEST_CATALOG_IMG} + pollInterval: 24h + insecureSkipTLSVerify: true +EOF + + +kubectl apply -f - << EOF +apiVersion: olm.operatorframework.io/v1alpha1 +kind: ClusterExtension +metadata: + name: ${TEST_CLUSTER_EXTENSION_NAME} +spec: + installNamespace: default + packageName: prometheus + version: 1.0.0 + serviceAccount: + name: default +EOF + +kubectl wait --for=condition=Unpacked --timeout=60s ClusterCatalog $TEST_CLUSTER_CATALOG_NAME +kubectl wait --for=condition=Installed --timeout=60s ClusterExtension $TEST_CLUSTER_EXTENSION_NAME diff --git a/test/upgrade-e2e/post_upgrade_test.go b/test/upgrade-e2e/post_upgrade_test.go new file mode 100644 index 00000000..a55b0327 --- /dev/null +++ b/test/upgrade-e2e/post_upgrade_test.go @@ -0,0 +1,135 @@ +package upgradee2e + +import ( + "bufio" + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1" + + ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" +) + +func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { + t.Log("Starting checks after OLM upgrade") + ctx := context.Background() + + managerLabelSelector := labels.Set{"control-plane": "controller-manager"} + + t.Log("Checking that the controller-manager deployment is updated") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var managerDeployments appsv1.DeploymentList + assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) + assert.Len(ct, managerDeployments.Items, 1) + managerDeployment := managerDeployments.Items[0] + + assert.True(ct, + managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, + ) + }, time.Minute, time.Second) + + var managerPods corev1.PodList + t.Log("Waiting for only one controller-manager Pod to remain") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) + assert.Len(ct, managerPods.Items, 1) + }, time.Minute, time.Second) + + t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") + // Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes + logCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + substring := fmt.Sprintf(`"ClusterExtension": {"name":"%s"}`, testClusterExtensionName) + found, err := watchPodLogsForSubstring(logCtx, &managerPods.Items[0], "manager", substring) + require.NoError(t, err) + require.True(t, found) + + t.Log("Checking that the ClusterCatalog is unpacked") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var clusterCatalog catalogdv1alpha1.ClusterCatalog + assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) + cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogdv1alpha1.TypeUnpacked) + if !assert.NotNil(ct, cond) { + return + } + assert.Equal(ct, metav1.ConditionTrue, cond.Status) + assert.Equal(ct, catalogdv1alpha1.ReasonUnpackSuccessful, cond.Reason) + }, time.Minute, time.Second) + + t.Log("Checking that the ClusterExtension is installed") + var clusterExtension ocv1alpha1.ClusterExtension + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) + cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) + if !assert.NotNil(ct, cond) { + return + } + assert.Equal(ct, metav1.ConditionTrue, cond.Status) + assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) + assert.Contains(ct, cond.Message, "Instantiated bundle") + assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle) + assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle.Version) + }, time.Minute, time.Second) + + previousVersion := clusterExtension.Status.InstalledBundle.Version + + t.Log("Updating the ClusterExtension to change version") + // Make sure that after we upgrade OLM itself we can still reconcile old objects if we change them + clusterExtension.Spec.Version = "1.0.1" + require.NoError(t, c.Update(ctx, &clusterExtension)) + + t.Log("Checking that the ClusterExtension installs successfully") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) + cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) + if !assert.NotNil(ct, cond) { + return + } + assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) + assert.Contains(ct, cond.Message, "Instantiated bundle") + assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.ResolvedBundle) + assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.InstalledBundle) + assert.NotEqual(ct, previousVersion, clusterExtension.Status.InstalledBundle.Version) + }, time.Minute, time.Second) +} + +func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container, substring string) (bool, error) { + podLogOpts := corev1.PodLogOptions{ + Follow: true, + Container: container, + } + + req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream(ctx) + if err != nil { + return false, err + } + defer podLogs.Close() + + scanner := bufio.NewScanner(podLogs) + for scanner.Scan() { + line := scanner.Text() + + if strings.Contains(line, substring) { + return true, nil + } + } + + return false, scanner.Err() +} diff --git a/test/upgrade-e2e/upgrade_e2e_suite_test.go b/test/upgrade-e2e/upgrade_e2e_suite_test.go new file mode 100644 index 00000000..b976f31e --- /dev/null +++ b/test/upgrade-e2e/upgrade_e2e_suite_test.go @@ -0,0 +1,57 @@ +package upgradee2e + +import ( + "fmt" + "os" + "testing" + + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/operator-framework/operator-controller/pkg/scheme" +) + +const ( + testClusterCatalogNameEnv = "TEST_CLUSTER_CATALOG_NAME" + testClusterExtensionNameEnv = "TEST_CLUSTER_EXTENSION_NAME" +) + +var ( + c client.Client + kclientset kubernetes.Interface + + testClusterCatalogName string + testClusterExtensionName string +) + +func TestMain(m *testing.M) { + var ok bool + testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) + if !ok { + fmt.Printf("%q is not set", testClusterCatalogNameEnv) + os.Exit(1) + } + testClusterExtensionName, ok = os.LookupEnv(testClusterExtensionNameEnv) + if !ok { + fmt.Printf("%q is not set", testClusterExtensionNameEnv) + os.Exit(1) + } + + cfg := ctrl.GetConfigOrDie() + + var err error + c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + if err != nil { + fmt.Printf("failed to create client: %s\n", err) + os.Exit(1) + } + + kclientset, err = kubernetes.NewForConfig(cfg) + if err != nil { + fmt.Printf("failed to create kubernetes clientset: %s\n", err) + os.Exit(1) + } + + os.Exit(m.Run()) +}