Add e2e test for upgrade

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>

test/e2e/common.go

@@ -40,11 +40,11 @@ import (
 const (
 	KubernetesVersionManagement     = "KUBERNETES_VERSION_MANAGEMENT"
 	KubernetesVersion               = "KUBERNETES_VERSION"
-	KubernetesVersionUpgradeFrom    = "KUBERNETES_VERSION_UPGRADE_FROM"
 	KubernetesVersionUpgradeTo      = "KUBERNETES_VERSION_UPGRADE_TO"
 	CPMachineTemplateUpgradeTo      = "CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO"
 	WorkersMachineTemplateUpgradeTo = "WORKERS_MACHINE_TEMPLATE_UPGRADE_TO"
 	IPFamily                        = "IP_FAMILY"
+	KindImageVersion                = "KIND_IMAGE_VERSION"
 )
 
 func Byf(format string, a ...interface{}) {

test/e2e/config/e2e_conf.yaml

@@ -69,13 +69,13 @@ providers:
             new: "--leader-elect=false"
 
 variables:
-  KUBERNETES_VERSION_MANAGEMENT: "v1.24.4"
-  KUBERNETES_VERSION: "v1.24.4"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.28.0"
+  KUBERNETES_VERSION: "v1.28.1"
+  KIND_IMAGE_VERSION: "v1.28.0"
   NODE_DRAIN_TIMEOUT: "60s"
   CONFORMANCE_WORKER_MACHINE_COUNT: "2"
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.24.2"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.23.8"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.28.2"
   KUBERNETES_UPGRADE_OCI_IMAGE_ID: "${KUBERNETES_UPGRADE_OCI_IMAGE_ID}"
   IP_FAMILY: "IPv4"
   EXP_CLUSTER_RESOURCE_SET: "true"

test/e2e/data/infrastructure/cluster-template-docker.yaml

@@ -48,7 +48,8 @@ metadata:
   name:  "${CLUSTER_NAME}-control-plane"
 spec:
   template:
-    spec: {}
+    spec:
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -80,7 +81,8 @@ metadata:
   name: ${CLUSTER_NAME}-md-0
 spec:
   template:
-    spec: {}
+    spec:
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
 kind: RKE2ConfigTemplate
@@ -92,4 +94,4 @@ spec:
       agentConfig:
         version: ${KUBERNETES_VERSION}+rke2r1
         nodeAnnotations:
-          test: "true"
+          test: "true"

test/e2e/e2e_suite_test.go

@@ -31,6 +31,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
 
 	bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1"
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1"
@@ -104,6 +105,8 @@ func init() {
 func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)
 
+	ctrl.SetLogger(klog.Background())
+
 	RunSpecs(t, "caprke2-e2e")
 }
 

test/e2e/e2e_test.go

@@ -148,6 +148,38 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: result.ControlPlane,
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			By("Upgrading control plane and worker machines")
+			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
+				ClusterProxy: bootstrapClusterProxy,
+				ConfigCluster: clusterctl.ConfigClusterInput{
+					LogFolder:                clusterctlLogFolder,
+					ClusterctlConfigPath:     clusterctlConfigPath,
+					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
+					InfrastructureProvider:   "docker",
+					Flavor:                   "docker",
+					Namespace:                namespace.Name,
+					ClusterName:              clusterName,
+					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
+					ControlPlaneMachineCount: pointer.Int64Ptr(3),
+					WorkerMachineCount:       pointer.Int64Ptr(3),
+				},
+				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
+				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
+				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
+			}, result)
+
+			WaitForClusterToUpgrade(ctx, WaitForClusterToUpgradeInput{
+				Lister:              bootstrapClusterProxy.GetClient(),
+				ControlPlane:        result.ControlPlane,
+				MachineDeployments:  result.MachineDeployments,
+				VersionAfterUpgrade: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
+			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
+
+			WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{
+				Getter:       bootstrapClusterProxy.GetClient(),
+				ControlPlane: result.ControlPlane,
+			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
+
 			// TODO: this can be uncommented when control plane scaling down is working
 
 			// By("Scaling control plane nodes to 1")

test/e2e/helpers.go

@@ -20,6 +20,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -251,6 +252,43 @@ func WaitForMachineConditions(ctx context.Context, input WaitForMachineCondition
 	}, intervals...).Should(BeTrue(), framework.PrettyPrint(input.Machine)+"\n")
 }
 
+// WaitForClusterToUpgradeInput is the input for WaitForClusterToUpgrade.
+type WaitForClusterToUpgradeInput struct {
+	Lister              framework.Lister
+	ControlPlane        *controlplanev1.RKE2ControlPlane
+	MachineDeployments  []*clusterv1.MachineDeployment
+	VersionAfterUpgrade string
+}
+
+// WaitForClusterToUpgrade will wait for a cluster to be upgraded.
+func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeInput, intervals ...interface{}) {
+	By("Waiting for machines to update")
+
+	totallMachineCount := *input.ControlPlane.Spec.Replicas
+	for _, md := range input.MachineDeployments {
+		totallMachineCount += *md.Spec.Replicas
+	}
+
+	Eventually(func() (bool, error) {
+		machineList := &clusterv1.MachineList{}
+		if err := input.Lister.List(ctx, machineList); err != nil {
+			return false, fmt.Errorf("failed to list machines: %w", err)
+		}
+
+		if len(machineList.Items) != int(totallMachineCount) { // not all replicas are created
+			return false, nil
+		}
+
+		for _, machine := range machineList.Items {
+			if machine.Spec.Version != nil && *machine.Spec.Version != input.VersionAfterUpgrade {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	}, intervals...).Should(BeTrue(), framework.PrettyPrint(input.ControlPlane)+"\n")
+}
+
 func setDefaults(input *ApplyClusterTemplateAndWaitInput) {
 	if input.WaitForControlPlaneInitialized == nil {
 		input.WaitForControlPlaneInitialized = func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) {