diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index cb4615ac5ffc..514383e8cae3 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -161,6 +161,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo), DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo), MachineDeployments: clusterResources.MachineDeployments, + MachinePools: clusterResources.MachinePools, KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go index 9a5f9f7d45b2..56b53bd9728e 100644 --- a/test/e2e/cluster_upgrade_runtimesdk.go +++ b/test/e2e/cluster_upgrade_runtimesdk.go @@ -221,6 +221,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl Cluster: clusterResources.Cluster, ControlPlane: clusterResources.ControlPlane, MachineDeployments: clusterResources.MachineDeployments, + MachinePools: clusterResources.MachinePools, KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), @@ -233,7 +234,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) }, - PreWaitForMachineDeploymentToBeUpgraded: func() { + PreWaitForWorkersToBeUpgraded: func() { machineSetPreflightChecksTestHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterRef) diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml index 128f1d366cfe..8ad80b535b72 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml @@ -26,6 +26,15 @@ spec: nodeDeletionTimeout: "30s" replicas: ${WORKER_MACHINE_COUNT} failureDomain: fd4 + machinePools: + - class: "default-worker" + name: "mp-0" + nodeDeletionTimeout: "30s" + nodeVolumeDetachTimeout: "5m" + minReadySeconds: 5 + replicas: ${WORKER_MACHINE_COUNT} + failureDomains: + - fd4 variables: - name: kubeadmControlPlaneMaxSurge value: "1" diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml index b4d9b389f1d6..479086cae767 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml @@ -1,4 +1,3 @@ resources: - ../bases/crs.yaml - - ../bases/mp.yaml - cluster-runtimesdk.yaml diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml index 4a6a5032d533..44c449548234 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml @@ -1,4 +1,3 @@ resources: - ../bases/cluster-with-topology.yaml - ../bases/crs.yaml - - ../bases/mp.yaml diff --git a/test/e2e/self_hosted.go b/test/e2e/self_hosted.go index 3abc9a9a01f3..568c4cd40426 100644 --- a/test/e2e/self_hosted.go +++ b/test/e2e/self_hosted.go @@ -313,6 +313,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo), DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo), MachineDeployments: clusterResources.MachineDeployments, + MachinePools: clusterResources.MachinePools, KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), diff --git a/test/framework/cluster_topology_helpers.go b/test/framework/cluster_topology_helpers.go index 07509bed2346..927aea913555 100644 --- a/test/framework/cluster_topology_helpers.go +++ b/test/framework/cluster_topology_helpers.go @@ -28,6 +28,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework/internal/log" "sigs.k8s.io/cluster-api/util/patch" ) @@ -59,19 +60,21 @@ func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput // UpgradeClusterTopologyAndWaitForUpgradeInput is the input type for UpgradeClusterTopologyAndWaitForUpgrade. type UpgradeClusterTopologyAndWaitForUpgradeInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - ControlPlane *controlplanev1.KubeadmControlPlane - EtcdImageTag string - DNSImageTag string - MachineDeployments []*clusterv1.MachineDeployment - KubernetesUpgradeVersion string - WaitForMachinesToBeUpgraded []interface{} - WaitForKubeProxyUpgrade []interface{} - WaitForDNSUpgrade []interface{} - WaitForEtcdUpgrade []interface{} - PreWaitForControlPlaneToBeUpgraded func() - PreWaitForMachineDeploymentToBeUpgraded func() + ClusterProxy ClusterProxy + Cluster *clusterv1.Cluster + ControlPlane *controlplanev1.KubeadmControlPlane + EtcdImageTag string + DNSImageTag string + MachineDeployments []*clusterv1.MachineDeployment + MachinePools []*expv1.MachinePool + KubernetesUpgradeVersion string + WaitForMachinesToBeUpgraded []interface{} + WaitForMachinePoolToBeUpgraded []interface{} + WaitForKubeProxyUpgrade []interface{} + WaitForDNSUpgrade []interface{} + WaitForEtcdUpgrade []interface{} + PreWaitForControlPlaneToBeUpgraded func() + PreWaitForWorkersToBeUpgraded func() } // UpgradeClusterTopologyAndWaitForUpgrade upgrades a Cluster topology and waits for it to be upgraded. @@ -82,6 +85,7 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade") Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade") Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade") + Expect(input.MachinePools).ToNot(BeEmpty(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade") Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade") mgmtClient := input.ClusterProxy.GetClient() @@ -150,12 +154,12 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC }, input.WaitForEtcdUpgrade...) } - // Once the ControlPlane is upgraded we can run PreWaitForMachineDeploymentToBeUpgraded. + // Once the ControlPlane is upgraded we can run PreWaitForWorkersToBeUpgraded. // Note: This can e.g. be used to verify the AfterControlPlaneUpgrade lifecycle hook is executed // and blocking correctly. - if input.PreWaitForMachineDeploymentToBeUpgraded != nil { - log.Logf("Calling PreWaitForMachineDeploymentToBeUpgraded") - input.PreWaitForMachineDeploymentToBeUpgraded() + if input.PreWaitForWorkersToBeUpgraded != nil { + log.Logf("Calling PreWaitForWorkersToBeUpgraded") + input.PreWaitForWorkersToBeUpgraded() } for _, deployment := range input.MachineDeployments { @@ -171,4 +175,19 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC }, input.WaitForMachinesToBeUpgraded...) } } + + for _, pool := range input.MachinePools { + if *pool.Spec.Replicas > 0 { + log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s to be upgraded to %s", + klog.KObj(pool), input.KubernetesUpgradeVersion) + WaitForMachinePoolInstancesToBeUpgraded(ctx, WaitForMachinePoolInstancesToBeUpgradedInput{ + Getter: mgmtClient, + WorkloadClusterGetter: input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient(), + Cluster: input.Cluster, + MachineCount: int(*pool.Spec.Replicas), + KubernetesUpgradeVersion: input.KubernetesUpgradeVersion, + MachinePool: pool, + }, input.WaitForMachinePoolToBeUpgraded...) + } + } }