Add bmo upgrade test

.github/workflows/e2e-fixture-test.yml

@@ -30,6 +30,7 @@ jobs:
 
     - name: Set Up Environment and Run BMO e2e Tests
       env:
-        E2E_CONF_FILE: ${GITHUB_WORKSPACE}/test/e2e/config/fixture.yaml
+        E2E_CONF_FILE: ${{ github.workspace }}/test/e2e/config/fixture.yaml
         USE_EXISTING_CLUSTER: "false"
+        BMO_RELEASE_BRANCH: "release-0.4"
       run: make test-e2e

Makefile

@@ -104,6 +104,9 @@ unit-verbose: ## Run unit tests with verbose output
 
 ARTIFACTS ?= ${ROOT_DIR}/test/e2e/_artifacts
 
+e2e-ensure-upgrade-kustomization: 
+	./hack/e2e/ensure_kustomization_paths.sh
+
 .PHONY: test-e2e
 test-e2e: $(GINKGO) ## Run the end-to-end tests
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
@@ -112,7 +115,7 @@ test-e2e: $(GINKGO) ## Run the end-to-end tests
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) test/e2e -- \
 		-e2e.config="$(E2E_CONF_FILE)" -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.artifacts-folder="$(ARTIFACTS)"
-
+	
 ## --------------------------------------
 ## Linter Targets
 ## --------------------------------------

config/overlays/e2e-release-0.4/ironic.env

@@ -0,0 +1,9 @@
+HTTP_PORT=6180
+PROVISIONING_IP=192.168.222.199
+DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=http://192.168.222.199:6385/v1/
+CACHEURL=http://192.168.222.199/images
+IRONIC_FAST_TRACK=true
+IRONIC_KERNEL_PARAMS=console=ttyS0
+IRONIC_INSPECTOR_VLAN_INTERFACES=all

config/overlays/e2e-release-0.4/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.4
+- https://github.com/metal3-io/baremetal-operator/config/default?ref=release-0.4
+
+configMapGenerator:
+- behavior: replace
+  envs:
+  - ironic.env
+  name: ironic
+
+patches:
+- patch: |
+    # Don't try to pull again the pre-loaded image
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: controller-manager
+
+images:
+- name: quay.io/metal3-io/baremetal-operator
+  newTag: release-0.4

config/overlays/fixture-release-0.4/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.4
+- https://github.com/metal3-io/baremetal-operator/config/default?ref=release-0.4
+
+patches:
+- patch: |
+    # Enable test mode (fixture provider instead of ironic)
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --test-mode
+    # Don't try to pull again the pre-loaded image
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: controller-manager
+
+images:
+- name: quay.io/metal3-io/baremetal-operator
+  newTag: release-0.4

hack/ci-e2e.sh

@@ -11,6 +11,7 @@
 set -eux
 
 REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
+BMO_IMAGE="quay.io/metal3-io/baremetal-operator:e2e"
 
 cd "${REPO_ROOT}" || exit 1
 
@@ -25,24 +26,35 @@ export PATH="${PATH}:/usr/local/go/bin"
 "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
 
 # Build the container image with e2e tag (used in tests)
-IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
-
-# Set up minikube
-minikube start --driver=kvm2
+IMG="${BMO_IMAGE}" make docker
 
 virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
 virsh -c qemu:///system net-start baremetal-e2e
-# Attach baremetal-e2e interface to minikube with specific mac.
-# This will give minikube a known reserved IP address that we can use for Ironic
-virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
-  --model virtio --source baremetal-e2e --type network --config
 
-# Restart minikube to apply the changes
-minikube stop
-minikube start
+# kubeconfigs=("${HOME}/.kube/config" "/tmp/kubeconfig-bmo-e2e-upgrade")
+profiles=("bmo-e2e" "bmo-e2e-upgrade")
+
+# Set up minikube
+for profile in "${profiles[@]}"; do
+  # export KUBECONFIG="${kubeconfigs[$index]}"
+  minikube start --profile "${profile}" --driver=kvm2
+
+  # Attach baremetal-e2e interface to minikube with specific mac.
+  # This will give minikube a known reserved IP address that we can use for Ironic
+  random_mac=$(hexdump -n 6 -ve '1/1 "%.2x "' /dev/random | awk -v a="2,6,a,e" -v r="$RANDOM" 'BEGIN{srand(r);}NR==1{split(a,b,",");r=int(rand()*4+1);printf "%s%s:%s:%s:%s:%s:%s\n",substr($1,0,1),b[r],$2,$3,$4,$5,$6}')
+  virsh -c qemu:///system attach-interface --domain "${profile}" --mac="${random_mac}" \
+    --model virtio --source baremetal-e2e --type network --config
+
+  # Restart minikube to apply the changes
+  minikube stop --profile "${profile}"
+  minikube start --profile "${profile}"
+
+  # Load the local BMO image into it
+  minikube --profile "${profile}" image load "${BMO_IMAGE}"
+done
 
-# Load the BMO e2e image into it
-minikube image load quay.io/metal3-io/baremetal-operator:e2e
+# export KUBECONFIG="${kubeconfigs[0]}"
+# export KUBECONFIG_E2E_UPGRADE="${kubeconfigs[1]}"
 
 # Create libvirt domain
 VM_NAME="bmo-e2e-0"
@@ -126,6 +138,8 @@ docker run --name image-server-e2e -d \
 set +e
 
 # Run the e2e tests
+export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
+export USE_EXISTING_CLUSTER="true"
 make test-e2e
 test_status="$?"
 

hack/clean-e2e.sh

@@ -3,7 +3,8 @@
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
-minikube delete
+minikube delete --profile bmo-e2e
+minikube delete --profile bmo-e2e-upgrade
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools

test/e2e/common.go

@@ -16,6 +16,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/clientcmd"
 
 	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
 
@@ -296,3 +297,41 @@ func CreateBMHCredentialsSecret(ctx context.Context, client client.Client, secre
 
 	return nil
 }
+
+func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProxy, name string, namespace string, desiredGeneration int64) bool {
+	clientSet := clusterProxy.GetClientSet()
+	deploy, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	Expect(err).To(BeNil())
+	if deploy != nil {
+		// When the number of replicas is equal to the number of available and updated
+		// replicas, we know that only "new" pods are running. When we also
+		// have the desired number of replicas and a new enough generation, we
+		// know that the rollout is complete.
+		return (deploy.Status.UpdatedReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.AvailableReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.Replicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.ObservedGeneration >= desiredGeneration)
+	}
+	return false
+}
+
+func SwitchKubeconfigContext(kubeConfigFile, contextName string) error {
+	config, err := clientcmd.LoadFromFile(kubeConfigFile)
+	if err != nil {
+		return err
+	}
+
+	_, exists := config.Contexts[contextName]
+	if !exists {
+		return errors.New(fmt.Sprintf("Context %s not existed in kubeconfig file %s", contextName, kubeConfigFile))
+	}
+
+	config.CurrentContext = contextName
+
+	err = clientcmd.WriteToFile(*config, kubeConfigFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

test/e2e/config/fixture.yaml

@@ -15,6 +15,8 @@ variables:
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/fixture"
+  BMO_RELEASE_BRANCH: "release-0.4"
+  BMO_UPGRADE_FROM_KUSTOMIZATION: "../../config/overlays/fixture-release-0.4"
   EXPECTED_HOST_NAME: ""
   # Test credentials. The tests will create a BMH with these.
   # For the fixture provider it does not matter much what we put here.

test/e2e/config/ironic.yaml

@@ -18,6 +18,8 @@ variables:
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
+  BMO_UPGRADE_FROM_KUSTOMIZATION: "../../config/overlays/e2e-release-0.4"
+  BMO_RELEASE_BRANCH: "release-0.4"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
   EXPECTED_HOST_NAME: "localhost.localdomain"
   # Test credentials. The tests will create a BMH with these.
@@ -49,3 +51,5 @@ intervals:
   default/wait-deprovisioning: ["1m", "10ms"]
   default/wait-deleted: ["20s", "10ms"]
   default/wait-secret-deletion: ["1m", "1s"]
+  upgrade/wait-provisioning: ["1m", "1s"]
+  upgrade/wait-provisioned: ["10m", "1s"]

test/e2e/e2e_suite_test.go

@@ -77,6 +77,8 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		if kubeconfigPath == "" {
 			kubeconfigPath = os.Getenv("HOME") + "/.kube/config"
 		}
+		err := SwitchKubeconfigContext(kubeconfigPath, "bmo-e2e")
+		Expect(err).NotTo(HaveOccurred())
 	} else {
 		clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{
 			Name:   "bmo-e2e",
@@ -89,6 +91,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 
 	scheme := runtime.NewScheme()
 	framework.TryAddDefaultSchemes(scheme)
+
 	clusterProxy := framework.NewClusterProxy("bmo-e2e", kubeconfigPath, scheme)
 	Expect(clusterProxy).ToNot(BeNil(), "Failed to get a cluster proxy")