diff --git a/test/e2e/common.go b/test/e2e/common.go index ced7bf5b..a91b0750 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -40,11 +40,11 @@ import ( const ( KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT" KubernetesVersion = "KUBERNETES_VERSION" - KubernetesVersionUpgradeFrom = "KUBERNETES_VERSION_UPGRADE_FROM" KubernetesVersionUpgradeTo = "KUBERNETES_VERSION_UPGRADE_TO" CPMachineTemplateUpgradeTo = "CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO" WorkersMachineTemplateUpgradeTo = "WORKERS_MACHINE_TEMPLATE_UPGRADE_TO" IPFamily = "IP_FAMILY" + KindImageVersion = "KIND_IMAGE_VERSION" ) func Byf(format string, a ...interface{}) { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index ce2f2730..0928ffe7 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -69,13 +69,13 @@ providers: new: "--leader-elect=false" variables: - KUBERNETES_VERSION_MANAGEMENT: "v1.24.4" - KUBERNETES_VERSION: "v1.24.4" + KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" + KUBERNETES_VERSION: "v1.28.1" + KIND_IMAGE_VERSION: "v1.28.0" NODE_DRAIN_TIMEOUT: "60s" CONFORMANCE_WORKER_MACHINE_COUNT: "2" CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1" - KUBERNETES_VERSION_UPGRADE_TO: "v1.24.2" - KUBERNETES_VERSION_UPGRADE_FROM: "v1.23.8" + KUBERNETES_VERSION_UPGRADE_TO: "v1.28.2" KUBERNETES_UPGRADE_OCI_IMAGE_ID: "${KUBERNETES_UPGRADE_OCI_IMAGE_ID}" IP_FAMILY: "IPv4" EXP_CLUSTER_RESOURCE_SET: "true" diff --git a/test/e2e/data/infrastructure/cluster-template-docker.yaml b/test/e2e/data/infrastructure/cluster-template-docker.yaml index 3b70f2dc..017e2189 100644 --- a/test/e2e/data/infrastructure/cluster-template-docker.yaml +++ b/test/e2e/data/infrastructure/cluster-template-docker.yaml @@ -48,7 +48,8 @@ metadata: name: "${CLUSTER_NAME}-control-plane" spec: template: - spec: {} + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -80,7 +81,8 @@ metadata: name: ${CLUSTER_NAME}-md-0 spec: template: - spec: {} + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 kind: RKE2ConfigTemplate @@ -92,4 +94,4 @@ spec: agentConfig: version: ${KUBERNETES_VERSION}+rke2r1 nodeAnnotations: - test: "true" \ No newline at end of file + test: "true" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 5fa50883..651d4698 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -31,6 +31,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1" controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1" @@ -104,6 +105,8 @@ func init() { func TestE2E(t *testing.T) { RegisterFailHandler(Fail) + ctrl.SetLogger(klog.Background()) + RunSpecs(t, "caprke2-e2e") } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 374d46cb..12641697 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -148,6 +148,38 @@ var _ = Describe("Workload cluster creation", func() { ControlPlane: result.ControlPlane, }, e2eConfig.GetIntervals(specName, "wait-control-plane")...) + By("Upgrading control plane and worker machines") + ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: "docker", + Flavor: "docker", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo), + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(3), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + + WaitForClusterToUpgrade(ctx, WaitForClusterToUpgradeInput{ + Lister: bootstrapClusterProxy.GetClient(), + ControlPlane: result.ControlPlane, + MachineDeployments: result.MachineDeployments, + VersionAfterUpgrade: e2eConfig.GetVariable(KubernetesVersionUpgradeTo), + }, e2eConfig.GetIntervals(specName, "wait-control-plane")...) + + WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{ + Getter: bootstrapClusterProxy.GetClient(), + ControlPlane: result.ControlPlane, + }, e2eConfig.GetIntervals(specName, "wait-control-plane")...) + // TODO: this can be uncommented when control plane scaling down is working // By("Scaling control plane nodes to 1") diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index a3b08b03..45894834 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -20,6 +20,7 @@ package e2e import ( "context" + "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -251,6 +252,43 @@ func WaitForMachineConditions(ctx context.Context, input WaitForMachineCondition }, intervals...).Should(BeTrue(), framework.PrettyPrint(input.Machine)+"\n") } +// WaitForClusterToUpgradeInput is the input for WaitForClusterToUpgrade. +type WaitForClusterToUpgradeInput struct { + Lister framework.Lister + ControlPlane *controlplanev1.RKE2ControlPlane + MachineDeployments []*clusterv1.MachineDeployment + VersionAfterUpgrade string +} + +// WaitForClusterToUpgrade will wait for a cluster to be upgraded. +func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeInput, intervals ...interface{}) { + By("Waiting for machines to update") + + totallMachineCount := *input.ControlPlane.Spec.Replicas + for _, md := range input.MachineDeployments { + totallMachineCount += *md.Spec.Replicas + } + + Eventually(func() (bool, error) { + machineList := &clusterv1.MachineList{} + if err := input.Lister.List(ctx, machineList); err != nil { + return false, fmt.Errorf("failed to list machines: %w", err) + } + + if len(machineList.Items) != int(totallMachineCount) { // not all replicas are created + return false, nil + } + + for _, machine := range machineList.Items { + if machine.Spec.Version != nil && *machine.Spec.Version != input.VersionAfterUpgrade { + return false, nil + } + } + + return true, nil + }, intervals...).Should(BeTrue(), framework.PrettyPrint(input.ControlPlane)+"\n") +} + func setDefaults(input *ApplyClusterTemplateAndWaitInput) { if input.WaitForControlPlaneInitialized == nil { input.WaitForControlPlaneInitialized = func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) {