🌱 Add MachinePools to Topology Quickstart E2E Templates

docs/book/src/reference/owner_references.md

@@ -51,7 +51,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
 |----------------------------|--------------------|------------|--------------------------|
 | ClusterResourcesSet        | None               |            |                          |
 | ClusterResourcesSetBinding | ClusterResourceSet | no         | May have many CRS owners |
-| MachinePool                | Cluster            | unknown    | Not tested in e2e        |
+| MachinePool                | Cluster            | no         |                          |
 
 
 ## KubeadmControlPlane types
@@ -65,7 +65,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
 | type                  | Owner        | Controller | Note                                            |
 |-----------------------|--------------|------------|-------------------------------------------------|
 | KubeadmConfig         | Machine      | yes        | When created for Machine                        |
-| KubeadmConfig         | MachinePool  | unknown    | When created for MachinePool. Not tested in e2e |
+| KubeadmConfig         | MachinePool  | yes        | When created for MachinePool                    |
 | KubeadmConfigTemplate | Cluster      | no         | When referenced in MachineDeployment spec       |
 | KubeadmConfigTemplate | ClusterClass | no         | When referenced in ClusterClass                 |
 
@@ -77,6 +77,6 @@ These owner references are almost all tested in an [end-to-end test](https://git
 | InfrastructureMachineTemplate | ClusterClass | no         | When referenced in a ClusterClass           |
 | InfrastructureCluster         | Cluster      | yes        |                                             |
 | InfrastructureClusterTemplate | ClusterClass | no         |                                             | 
-| InfrastructureMachinePool     | MachinePool  | unknown    | Not tested in e2e                           |
+| InfrastructureMachinePool     | MachinePool  | yes        |                                             |
 
 

test/e2e/data/infrastructure-docker/main/bases/cluster-with-topology.yaml

@@ -47,6 +47,22 @@ spec:
           rollingUpdate:
             maxSurge: "20%"
             maxUnavailable: 0
+      machinePools:
+      - class: "default-worker"
+        name: "mp-0"
+        metadata:
+          labels:
+            Cluster.topology.machinePool.label: "Cluster.topology.machinePool.labelValue"
+            # Note: this label is propagated to Nodes.
+            Cluster.topology.machinePool.label.node.cluster.x-k8s.io: "Cluster.topology.machinePool.nodeLabelValue"
+          annotations:
+            Cluster.topology.machinePool.annotation: "Cluster.topology.machinePool.annotationValue"
+        nodeDeletionTimeout: "30s"
+        nodeVolumeDetachTimeout: "5m"
+        minReadySeconds: 5
+        replicas: ${WORKER_MACHINE_COUNT}
+        failureDomains:
+        - fd4
     variables:
       # We set an empty value to use the default tag kubeadm init is using.
     - name: etcdImageTag

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

@@ -36,6 +36,21 @@ spec:
             apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
             kind: DockerMachineTemplate
             name: quick-start-default-worker-machinetemplate
+    machinePools:
+    - class: default-worker
+      namingStrategy:
+        template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: quick-start-default-worker-bootstraptemplate
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: DockerMachinePoolTemplate
+            name: quick-start-default-worker-machinepooltemplate
   variables:
   # This variable is not used in any patch, but is here to ensure the variable discovered from the runtime hook is correctly
   # used.
@@ -122,6 +137,18 @@ spec:
       - containerPath: "/var/run/docker.sock"
         hostPath: "/var/run/docker.sock"
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachinePoolTemplate
+metadata:
+  name: quick-start-default-worker-machinepooltemplate
+spec:
+  template:
+    spec:
+      template:
+        extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml

@@ -42,7 +42,7 @@ spec:
           ref:
             apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
             kind: KubeadmConfigTemplate
-            name: quick-start-default-worker-bootstraptemplate
+            name: quick-start-md-default-worker-bootstraptemplate
         infrastructure:
           ref:
             apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -54,6 +54,24 @@ spec:
         - type: e2e.remediation.condition
           status: "False"
           timeout: 20s
+    machinePools:
+    - class: default-worker
+      template:
+        metadata:
+          labels:
+            ClusterClass.machinePool.label: "ClusterClass.machinePool.labelValue"
+          annotations:
+            ClusterClass.machinePool.annotation: "ClusterClass.machinePool.annotationValue"
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: quick-start-mp-default-worker-bootstraptemplate
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: DockerMachinePoolTemplate
+            name: quick-start-default-worker-machinepooltemplate
   variables:
   - name: lbImageRepository
     required: true
@@ -182,6 +200,19 @@ spec:
             valueFrom:
               template: |
                 kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachinePoolTemplate
+          matchResources:
+            machinePoolClass:
+              names:
+                - default-worker
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/template/customImage"
+            valueFrom:
+              template: |
+                kindest/node:{{ .builtin.machinePool.version | replace "+" "_" }}
   - name: preloadImages
     description: |
       Sets the container images to preload to the node that is used for running dockerMachines.
@@ -201,6 +232,24 @@ spec:
         path: "/spec/template/spec/preLoadImages"
         valueFrom:
           variable: preLoadImages
+  - name: preloadImagesMP
+    description: |
+      Sets the container images to preload to the node that is used for running dockerMachines.
+      This is especially required for self-hosted e2e tests to ensure the required controller images to be available
+      and reduce load to public registries.
+    definitions:
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachinePoolTemplate
+          matchResources:
+            machinePoolClass:
+              names:
+                - default-worker
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/template/preLoadImages"
+            valueFrom:
+              variable: preLoadImages
   - name: kubeadmControlPlaneMaxSurge
     description: "Sets the maxSurge value used for rolloutStrategy in the KubeadmControlPlane."
     enabledIf: '{{ ne .kubeadmControlPlaneMaxSurge "" }}'
@@ -264,6 +313,18 @@ spec:
             path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
             value:
               cloud-provider: "external"
+      - selector:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          matchResources:
+            machinePoolClass:
+              names:
+                - '*-worker'
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
+            value:
+              cloud-provider: "external"
   - name: localEndpointIPv6
     enabledIf: "{{ .ipv6Primary }}"
     description: "Configures KCP to use IPv6 for its localAPIEndpoint."
@@ -440,10 +501,31 @@ spec:
       - containerPath: "/var/run/docker.sock"
         hostPath: "/var/run/docker.sock"
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachinePoolTemplate
+metadata:
+  name: quick-start-default-worker-machinepooltemplate
+  labels:
+    InfraMachinePoolTemplate.machinePool.label: "InfraMachinePoolTemplate.machinePool.labelValue"
+  annotations:
+    InfraMachinePoolTemplate.machinePool.annotation: "InfraMachinePoolTemplate.machinePool.annotationValue"
+spec:
+  template:
+    metadata:
+      labels:
+        InfraMachinePoolTemplate.machinePool.template.label: "InfraMachinePoolTemplate.machinePool.labelValue"
+      annotations:
+        InfraMachinePoolTemplate.machinePool.template.annotation: "InfraMachinePoolTemplate.machinePool.annotationValue"
+    spec:
+      template:
+        extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
-  name: quick-start-default-worker-bootstraptemplate
+  name: quick-start-md-default-worker-bootstraptemplate
   labels:
     BootstrapConfigTemplate.machineDeployment.label: "BootstrapConfigTemplate.machineDeployment.labelValue"
   annotations:
@@ -460,3 +542,24 @@ spec:
         nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
           kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
             eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: quick-start-mp-default-worker-bootstraptemplate
+  labels:
+    BootstrapConfigTemplate.machinePool.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
+  annotations:
+    BootstrapConfigTemplate.machinePool.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
+spec:
+  template:
+    metadata:
+      labels:
+        BootstrapConfigTemplate.machinePool.template.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
+      annotations:
+        BootstrapConfigTemplate.machinePool.template.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
+    spec:
+      joinConfiguration:
+        nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+          kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
+            eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'

test/framework/ownerreference_helpers.go

@@ -183,7 +183,6 @@ var (
 	clusterResourceSetBindingKind = "ClusterResourceSetBinding"
 	machinePoolKind               = "MachinePool"
 
-	machinePoolOwner      = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String()}
 	machinePoolController = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String(), Controller: pointer.Bool(true)}
 
 	clusterResourceSetOwner = metav1.OwnerReference{Kind: clusterResourceSetKind, APIVersion: addonsv1.GroupVersion.String()}
@@ -270,7 +269,7 @@ var (
 var KubeadmBootstrapOwnerReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
 	kubeadmConfigKind: func(owners []metav1.OwnerReference) error {
 		// The KubeadmConfig must be owned and controlled by a Machine or MachinePool.
-		return HasOneOfExactOwners(owners, []metav1.OwnerReference{machineController}, []metav1.OwnerReference{machinePoolController})
+		return HasOneOfExactOwners(owners, []metav1.OwnerReference{machineController}, []metav1.OwnerReference{machinePoolController, clusterOwner})
 	},
 	kubeadmConfigTemplateKind: func(owners []metav1.OwnerReference) error {
 		// The KubeadmConfigTemplate must be owned by a ClusterClass.
@@ -280,11 +279,12 @@ var KubeadmBootstrapOwnerReferenceAssertions = map[string]func([]metav1.OwnerRef
 
 // Kinds for types in the Docker infrastructure package.
 var (
-	dockerMachineKind         = "DockerMachine"
-	dockerMachineTemplateKind = "DockerMachineTemplate"
-	dockerMachinePoolKind     = "DockerMachinePool"
-	dockerClusterKind         = "DockerCluster"
-	dockerClusterTemplateKind = "DockerClusterTemplate"
+	dockerMachineKind             = "DockerMachine"
+	dockerMachineTemplateKind     = "DockerMachineTemplate"
+	dockerMachinePoolKind         = "DockerMachinePool"
+	dockerMachinePoolTemplateKind = "DockerMachinePoolTemplate"
+	dockerClusterKind             = "DockerCluster"
+	dockerClusterTemplateKind     = "DockerClusterTemplate"
 )
 
 // DockerInfraOwnerReferenceAssertions maps Docker Infrastructure types to functions which return an error if the passed
@@ -310,8 +310,12 @@ var DockerInfraOwnerReferenceAssertions = map[string]func([]metav1.OwnerReferenc
 		return HasExactOwners(owners, clusterClassOwner)
 	},
 	dockerMachinePoolKind: func(owners []metav1.OwnerReference) error {
-		// DockerMachinePool must be owned by a MachinePool.
-		return HasExactOwners(owners, machinePoolOwner)
+		// DockerMachinePool must be owned and controlled by a MachinePool.
+		return HasExactOwners(owners, machinePoolController, clusterOwner)
+	},
+	dockerMachinePoolTemplateKind: func(owners []metav1.OwnerReference) error {
+		// DockerMachinePoolTemplate must be owned by a ClusterClass.
+		return HasExactOwners(owners, clusterClassOwner)
 	},
 }
 
@@ -328,7 +332,7 @@ func HasExactOwners(gotOwners []metav1.OwnerReference, wantOwners ...metav1.Owne
 	sort.Strings(wantComparable)
 
 	if !reflect.DeepEqual(gotComparable, wantComparable) {
-		return fmt.Errorf("wanted %v, actual %v", wantOwners, gotOwners)
+		return fmt.Errorf("wanted %v, actual %v", wantComparable, gotComparable)
 	}
 	return nil
 }