Add bmo upgrade test

.github/workflows/e2e-fixture-test.yml

@@ -32,4 +32,5 @@ jobs:
       env:
         E2E_CONF_FILE: ${GITHUB_WORKSPACE}/test/e2e/config/fixture.yaml
         USE_EXISTING_CLUSTER: "false"
+        BMO_RELEASE_BRANCH: "release-0.4"
       run: make test-e2e

Makefile

@@ -104,6 +104,9 @@ unit-verbose: ## Run unit tests with verbose output
 
 ARTIFACTS ?= ${ROOT_DIR}/test/e2e/_artifacts
 
+e2e-ensure-upgrade-kustomization: 
+	./hack/e2e/ensure_kustomization_paths.sh
+
 .PHONY: test-e2e
 test-e2e: $(GINKGO) ## Run the end-to-end tests
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
@@ -112,6 +115,8 @@ test-e2e: $(GINKGO) ## Run the end-to-end tests
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) test/e2e -- \
 		-e2e.config="$(E2E_CONF_FILE)" -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.artifacts-folder="$(ARTIFACTS)"
+
+test-e2e: e2e-ensure-upgrade-kustomization
 
 ## --------------------------------------
 ## Linter Targets

hack/ci-e2e.sh

@@ -11,6 +11,9 @@
 set -eux
 
 REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
+BMO_IMAGE_NAME="quay.io/metal3-io/baremetal-operator"
+
+export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
 
 cd "${REPO_ROOT}" || exit 1
 
@@ -25,24 +28,31 @@ export PATH="${PATH}:/usr/local/go/bin"
 "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
 
 # Build the container image with e2e tag (used in tests)
-IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
-
-# Set up minikube
-minikube start --driver=kvm2
+IMG="${BMO_IMAGE_NAME}:e2e" make docker
 
 virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
 virsh -c qemu:///system net-start baremetal-e2e
-# Attach baremetal-e2e interface to minikube with specific mac.
-# This will give minikube a known reserved IP address that we can use for Ironic
-virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
-  --model virtio --source baremetal-e2e --type network --config
 
-# Restart minikube to apply the changes
-minikube stop
-minikube start
+export KUBECONFIG_E2E="/tmp/kubeconfig-bmo-e2e"
+export KUBECONFIG_E2E_UPGRADE="/tmp/kubeconfig-bmo-e2e-upgrade"
+
+# Set up minikube
+for profile in "bmo-e2e" "bmo-e2e-upgrade"; do
+  export KUBECONFIG="/tmp/kubeconfig-${profile}"
+  minikube start --profile $profile --driver=kvm2
+
+  # Attach baremetal-e2e interface to minikube with specific mac.
+  # This will give minikube a known reserved IP address that we can use for Ironic
+  virsh -c qemu:///system attach-interface --domain "${profile}" --mac="52:54:00:6c:3c:01" \
+    --model virtio --source baremetal-e2e --type network --config
+
+  # Restart minikube to apply the changes
+  minikube stop --profile "${profile}"
+  minikube start --profile "${profile}"
 
-# Load the BMO e2e image into it
-minikube image load quay.io/metal3-io/baremetal-operator:e2e
+  # Load the local BMO image into it
+  minikube --profile "${profile}" image load "${BMO_IMAGE_NAME}:e2e"
+done
 
 # Create libvirt domain
 VM_NAME="bmo-e2e-0"
@@ -126,6 +136,8 @@ docker run --name image-server-e2e -d \
 set +e
 
 # Run the e2e tests
+export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
+export USE_EXISTING_CLUSTER="true"
 make test-e2e
 test_status="$?"
 

hack/clean-e2e.sh

@@ -3,7 +3,8 @@
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
-minikube delete
+minikube delete --profile bmo-e2e
+minikube delete --profile bmo-e2e-upgrade
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools

hack/e2e/ensure_kustomization_paths.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+set -eux
+
+REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/../..")
+
+BMO_IMAGE_NAME="quay.io/metal3-io/baremetal-operator"
+BMO_RELEASE_BRANCH="${BMO_RELEASE_BRANCH:-release-0.4}"
+BMO_RELEASE_ROOT="/tmp/bmo-${BMO_RELEASE_BRANCH}"
+BMO_ORIGIN_ADDRESS="https://github.com/metal3-io/baremetal-operator.git"
+
+BMO_E2E_KUSTOMIZATION_PATH=$(grep BMO_KUSTOMIZATION "${E2E_CONF_FILE}" | awk -F: '{ print $2 }' | tr -d '" ')
+if [[ "${BMO_E2E_KUSTOMIZATION_PATH}" == "" ]]; then
+  echo "BMO_KUSTOMIZATION not defined in ${E2E_CONF_FILE}"
+  exit 2
+fi
+
+BMO_E2E_KUSTOMIZATION=$(realpath "${REPO_ROOT}/test/e2e/${BMO_E2E_KUSTOMIZATION_PATH}")
+if [[ ! -d "${BMO_E2E_KUSTOMIZATION}" ]]; then
+  echo "Kustomization path not exists: ${BMO_E2E_KUSTOMIZATION}"
+  exit 2
+fi
+
+BMO_E2E_KUSTOMIZATION_RELATIVE_PATH=$(realpath --relative-to="${REPO_ROOT}" "${BMO_E2E_KUSTOMIZATION}")
+
+if [[ ! -d "${BMO_RELEASE_ROOT}" ]]; then
+  git clone --branch "${BMO_RELEASE_BRANCH}" --single-branch "${BMO_ORIGIN_ADDRESS}" "${BMO_RELEASE_ROOT}"
+fi
+
+BMO_UPGRADE_FROM_KUSTOMIZATION="${BMO_RELEASE_ROOT}/${BMO_E2E_KUSTOMIZATION_RELATIVE_PATH}"
+
+if [[ ! -d "${BMO_UPGRADE_FROM_KUSTOMIZATION}" ]]; then
+  mkdir -p "${BMO_UPGRADE_FROM_KUSTOMIZATION}"
+  cp -r "${BMO_E2E_KUSTOMIZATION}/"* "${BMO_UPGRADE_FROM_KUSTOMIZATION}/"
+fi
+
+sed -i "s/newTag: e2e/newTag: ${BMO_RELEASE_BRANCH}/" "${BMO_UPGRADE_FROM_KUSTOMIZATION}/kustomization.yaml"

test/e2e/common.go

@@ -296,3 +296,20 @@ func CreateBMHCredentialsSecret(ctx context.Context, client client.Client, secre
 
 	return nil
 }
+
+func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProxy, name string, namespace string, desiredGeneration int64) bool {
+	clientSet := clusterProxy.GetClientSet()
+	deploy, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	Expect(err).To(BeNil())
+	if deploy != nil {
+		// When the number of replicas is equal to the number of available and updated
+		// replicas, we know that only "new" pods are running. When we also
+		// have the desired number of replicas and a new enough generation, we
+		// know that the rollout is complete.
+		return (deploy.Status.UpdatedReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.AvailableReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.Replicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.ObservedGeneration >= desiredGeneration)
+	}
+	return false
+}

test/e2e/config/fixture.yaml

@@ -15,6 +15,7 @@ variables:
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/fixture"
+  BMO_RELEASE_BRANCH: "release-0.4"
   EXPECTED_HOST_NAME: ""
   # Test credentials. The tests will create a BMH with these.
   # For the fixture provider it does not matter much what we put here.

test/e2e/config/ironic.yaml

@@ -18,6 +18,7 @@ variables:
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
+  BMO_RELEASE_BRANCH: "release-0.4"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
   EXPECTED_HOST_NAME: "localhost.localdomain"
   # Test credentials. The tests will create a BMH with these.
@@ -49,3 +50,5 @@ intervals:
   default/wait-deprovisioning: ["1m", "10ms"]
   default/wait-deleted: ["20s", "10ms"]
   default/wait-secret-deletion: ["1m", "1s"]
+  upgrade/wait-provisioning: ["1m", "1s"]
+  upgrade/wait-provisioned: ["10m", "1s"]

test/e2e/e2e_suite_test.go

@@ -73,7 +73,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	e2eConfig = LoadE2EConfig(configPath)
 
 	if useExistingCluster {
-		kubeconfigPath = os.Getenv("KUBECONFIG")
+		kubeconfigPath = os.Getenv("KUBECONFIG_E2E")
 		if kubeconfigPath == "" {
 			kubeconfigPath = os.Getenv("HOME") + "/.kube/config"
 		}