Merge pull request #9502 from willie-yao/cc-mp-upgrade

🌱 Add MachinePools to topology upgrade test
kubernetes-sigs · Sep 28, 2023 · 3b67cb7 · 3b67cb7
2 parents 8255149 + 02fbcb1
commit 3b67cb7
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 20 deletions.
diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go
@@ -161,6 +161,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 				EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
 				DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
 				MachineDeployments:          clusterResources.MachineDeployments,
+				MachinePools:                clusterResources.MachinePools,
 				KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
 				WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 				WaitForKubeProxyUpgrade:     input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),

diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go
@@ -221,6 +221,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
 			Cluster:                     clusterResources.Cluster,
 			ControlPlane:                clusterResources.ControlPlane,
 			MachineDeployments:          clusterResources.MachineDeployments,
+			MachinePools:                clusterResources.MachinePools,
 			KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
 			WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 			WaitForKubeProxyUpgrade:     input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
@@ -233,7 +234,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
 					input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
 					input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"))
 			},
-			PreWaitForMachineDeploymentToBeUpgraded: func() {
+			PreWaitForWorkersToBeUpgraded: func() {
 				machineSetPreflightChecksTestHandler(ctx,
 					input.BootstrapClusterProxy.GetClient(),
 					clusterRef)

diff --git a/...a/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml b/...a/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml
@@ -26,6 +26,15 @@ spec:
           nodeDeletionTimeout: "30s"
           replicas: ${WORKER_MACHINE_COUNT}
           failureDomain: fd4
+      machinePools:
+        - class: "default-worker"
+          name: "mp-0"
+          nodeDeletionTimeout: "30s"
+          nodeVolumeDetachTimeout: "5m"
+          minReadySeconds: 5
+          replicas: ${WORKER_MACHINE_COUNT}
+          failureDomains:
+          - fd4
     variables:
       - name: kubeadmControlPlaneMaxSurge
         value: "1"

diff --git a/...e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml b/...e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml
@@ -1,4 +1,3 @@
 resources:
   - ../bases/crs.yaml
-  - ../bases/mp.yaml
   - cluster-runtimesdk.yaml
diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml
@@ -1,4 +1,3 @@
 resources:
   - ../bases/cluster-with-topology.yaml
   - ../bases/crs.yaml
-  - ../bases/mp.yaml
diff --git a/test/e2e/self_hosted.go b/test/e2e/self_hosted.go
@@ -313,6 +313,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 				EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
 				DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
 				MachineDeployments:          clusterResources.MachineDeployments,
+				MachinePools:                clusterResources.MachinePools,
 				KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
 				WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 				WaitForKubeProxyUpgrade:     input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),

diff --git a/test/framework/cluster_topology_helpers.go b/test/framework/cluster_topology_helpers.go
@@ -28,6 +28,7 @@ import (
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
 	"sigs.k8s.io/cluster-api/util/patch"
 )
@@ -59,19 +60,21 @@ func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput
 
 // UpgradeClusterTopologyAndWaitForUpgradeInput is the input type for UpgradeClusterTopologyAndWaitForUpgrade.
 type UpgradeClusterTopologyAndWaitForUpgradeInput struct {
-	ClusterProxy                            ClusterProxy
-	Cluster                                 *clusterv1.Cluster
-	ControlPlane                            *controlplanev1.KubeadmControlPlane
-	EtcdImageTag                            string
-	DNSImageTag                             string
-	MachineDeployments                      []*clusterv1.MachineDeployment
-	KubernetesUpgradeVersion                string
-	WaitForMachinesToBeUpgraded             []interface{}
-	WaitForKubeProxyUpgrade                 []interface{}
-	WaitForDNSUpgrade                       []interface{}
-	WaitForEtcdUpgrade                      []interface{}
-	PreWaitForControlPlaneToBeUpgraded      func()
-	PreWaitForMachineDeploymentToBeUpgraded func()
+	ClusterProxy                       ClusterProxy
+	Cluster                            *clusterv1.Cluster
+	ControlPlane                       *controlplanev1.KubeadmControlPlane
+	EtcdImageTag                       string
+	DNSImageTag                        string
+	MachineDeployments                 []*clusterv1.MachineDeployment
+	MachinePools                       []*expv1.MachinePool
+	KubernetesUpgradeVersion           string
+	WaitForMachinesToBeUpgraded        []interface{}
+	WaitForMachinePoolToBeUpgraded     []interface{}
+	WaitForKubeProxyUpgrade            []interface{}
+	WaitForDNSUpgrade                  []interface{}
+	WaitForEtcdUpgrade                 []interface{}
+	PreWaitForControlPlaneToBeUpgraded func()
+	PreWaitForWorkersToBeUpgraded      func()
 }
 
 // UpgradeClusterTopologyAndWaitForUpgrade upgrades a Cluster topology and waits for it to be upgraded.
@@ -82,6 +85,7 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
 	Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade")
 	Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade")
 	Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
+	Expect(input.MachinePools).ToNot(BeEmpty(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
 	Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
 
 	mgmtClient := input.ClusterProxy.GetClient()
@@ -150,12 +154,12 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
 		}, input.WaitForEtcdUpgrade...)
 	}
 
-	// Once the ControlPlane is upgraded we can run PreWaitForMachineDeploymentToBeUpgraded.
+	// Once the ControlPlane is upgraded we can run PreWaitForWorkersToBeUpgraded.
 	// Note: This can e.g. be used to verify the AfterControlPlaneUpgrade lifecycle hook is executed
 	// and blocking correctly.
-	if input.PreWaitForMachineDeploymentToBeUpgraded != nil {
-		log.Logf("Calling PreWaitForMachineDeploymentToBeUpgraded")
-		input.PreWaitForMachineDeploymentToBeUpgraded()
+	if input.PreWaitForWorkersToBeUpgraded != nil {
+		log.Logf("Calling PreWaitForWorkersToBeUpgraded")
+		input.PreWaitForWorkersToBeUpgraded()
 	}
 
 	for _, deployment := range input.MachineDeployments {
@@ -171,4 +175,19 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
 			}, input.WaitForMachinesToBeUpgraded...)
 		}
 	}
+
+	for _, pool := range input.MachinePools {
+		if *pool.Spec.Replicas > 0 {
+			log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s to be upgraded to %s",
+				klog.KObj(pool), input.KubernetesUpgradeVersion)
+			WaitForMachinePoolInstancesToBeUpgraded(ctx, WaitForMachinePoolInstancesToBeUpgradedInput{
+				Getter:                   mgmtClient,
+				WorkloadClusterGetter:    input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient(),
+				Cluster:                  input.Cluster,
+				MachineCount:             int(*pool.Spec.Replicas),
+				KubernetesUpgradeVersion: input.KubernetesUpgradeVersion,
+				MachinePool:              pool,
+			}, input.WaitForMachinePoolToBeUpgraded...)
+		}
+	}
 }