Merge pull request #8868 from killianmuldoon/pr-1.4-kind-image-sha

[release-1.4] 🐛 Use known kindest/node image versions by sha for e2e upgrade tests

test/e2e/clusterctl_upgrade_test.go

@@ -36,6 +36,7 @@ var _ = Describe("When testing clusterctl upgrades (v0.3=>current)", func() {
 			InitWithBinary:            "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/clusterctl-{OS}-{ARCH}",
 			InitWithProvidersContract: "v1alpha3",
 			// CAPI v0.3.x does not work on Kubernetes >= v1.22.
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.3/bases.
 			InitWithKubernetesVersion: "v1.21.14",
 			WorkloadKubernetesVersion: "v1.22.17",
 			// CAPI does not work with Kubernetes < v1.22 if ClusterClass is enabled, so we have to disable it.
@@ -69,6 +70,7 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() {
 			SkipCleanup:               skipCleanup,
 			InitWithBinary:            "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}",
 			InitWithProvidersContract: "v1alpha4",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.4/bases.
 			InitWithKubernetesVersion: "v1.23.17",
 			WorkloadKubernetesVersion: "v1.23.17",
 			MgmtFlavor:                "topology",
@@ -107,10 +109,11 @@ var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
 			// runtime extension providers. If we don't do this the test will automatically
 			// try to deploy the latest version of our test-extension from docker.yaml.
 			InitWithRuntimeExtensionProviders: []string{},
-			InitWithKubernetesVersion:         "v1.23.17",
-			WorkloadKubernetesVersion:         "v1.23.17",
-			MgmtFlavor:                        "topology",
-			WorkloadFlavor:                    "",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.0/bases.
+			InitWithKubernetesVersion: "v1.23.17",
+			WorkloadKubernetesVersion: "v1.23.17",
+			MgmtFlavor:                "topology",
+			WorkloadFlavor:            "",
 			// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
 			PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
 				framework.ValidateOwnerReferencesOnUpdate(proxy, namespace,
@@ -145,9 +148,11 @@ var _ = Describe("When testing clusterctl upgrades (v1.2=>current)", func() {
 			// runtime extension providers. If we don't do this the test will automatically
 			// try to deploy the latest version of our test-extension from docker.yaml.
 			InitWithRuntimeExtensionProviders: []string{},
-			InitWithKubernetesVersion:         "v1.26.0",
+			InitWithKubernetesVersion:         "v1.26.4",
 			MgmtFlavor:                        "topology",
 			WorkloadFlavor:                    "",
+			InitWithProvidersContract:         "v1beta1",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.3/bases.
 			// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
 			PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
 				framework.ValidateOwnerReferencesOnUpdate(proxy, namespace,
@@ -182,9 +187,11 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.2=>cur
 			// runtime extension providers. If we don't do this the test will automatically
 			// try to deploy the latest version of our test-extension from docker.yaml.
 			InitWithRuntimeExtensionProviders: []string{},
-			InitWithKubernetesVersion:         "v1.26.0",
-			MgmtFlavor:                        "topology",
-			WorkloadFlavor:                    "topology",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.3/bases.
+			InitWithKubernetesVersion: "v1.26.4",
+			MgmtFlavor:                "topology",
+			WorkloadFlavor:            "topology",
+			InitWithProvidersContract: "v1beta1",
 			// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
 			PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
 				framework.ValidateOwnerReferencesOnUpdate(proxy, namespace,
@@ -210,8 +217,9 @@ var _ = Describe("When testing clusterctl upgrades (v1.3=>current)", func() {
 			SkipCleanup:               skipCleanup,
 			InitWithBinary:            "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.5/clusterctl-{OS}-{ARCH}",
 			InitWithProvidersContract: "v1beta1",
-			InitWithKubernetesVersion: "v1.26.3",
-			WorkloadKubernetesVersion: "v1.26.3",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.4/bases.
+			InitWithKubernetesVersion: "v1.26.4",
+			WorkloadKubernetesVersion: "v1.26.4",
 			MgmtFlavor:                "topology",
 			WorkloadFlavor:            "",
 			// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
@@ -239,8 +247,9 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.3=>cur
 			SkipCleanup:               skipCleanup,
 			InitWithBinary:            "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.5/clusterctl-{OS}-{ARCH}",
 			InitWithProvidersContract: "v1beta1",
-			InitWithKubernetesVersion: "v1.26.3",
-			WorkloadKubernetesVersion: "v1.26.3",
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.4/bases.
+			InitWithKubernetesVersion: "v1.26.4",
+			WorkloadKubernetesVersion: "v1.26.4",
 			MgmtFlavor:                "topology",
 			WorkloadFlavor:            "topology",
 			// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml

@@ -184,30 +184,86 @@ spec:
   - name: customImage
     description: "Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments."
     definitions:
-    - selector:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-        kind: DockerMachineTemplate
-        matchResources:
-          machineDeploymentClass:
-            names:
-            - default-worker
-      jsonPatches:
-      - op: add
-        path: "/spec/template/spec/customImage"
-        valueFrom:
-          template: |
-            kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }}
-    - selector:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-        kind: DockerMachineTemplate
-        matchResources:
-          controlPlane: true
-      jsonPatches:
-      - op: add
-        path: "/spec/template/spec/customImage"
-        valueFrom:
-          template: |
-            kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            machineDeploymentClass:
+              names:
+                - default-worker
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            valueFrom:
+              template: |
+                kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }}
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            controlPlane: true
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            valueFrom:
+              template: |
+                kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
+  - name: replaceImage-v1.23.17-machineDeployment
+    description: "Sets the container image for MD DockerMachineTemplates using Kubernetes v1.23.17."
+    enabledIf: '{{ semverCompare "v1.23.17" .builtin.machineDeployment.version }}'
+    definitions:
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            machineDeploymentClass:
+              names:
+                - default-worker
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
+  - name: replaceImage-v1.23.17-controlPlane
+    description: "Sets the container image for CP DockerMachineTemplates using Kubernetes v1.23.17."
+    enabledIf: '{{ semverCompare "v1.23.17" .builtin.controlPlane.version }}'
+    definitions:
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            controlPlane: true
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
+  - name: replaceImage-v1.21.14-machineDeployment
+    description: "Sets the container image for MD DockerMachineTemplates using Kubernetes v1.21.14."
+    enabledIf: '{{ semverCompare "v1.21.14" .builtin.machineDeployment.version }}'
+    definitions:
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            machineDeploymentClass:
+              names:
+                - default-worker
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            value: "kindest/node:v1.21.14@sha256:220cfafdf6e3915fbce50e13d1655425558cb98872c53f802605aa2fb2d569cf"
+  - name: replaceImage-v1.21.14-controlPlane
+    description: "Sets the container image for CP DockerMachineTemplates using Kubernetes v1.21.14."
+    enabledIf: '{{ semverCompare "v1.21.14" .builtin.controlPlane.version }}'
+    definitions:
+      - selector:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+          kind: DockerMachineTemplate
+          matchResources:
+            controlPlane: true
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/customImage"
+            value: "kindest/node:v1.21.14@sha256:220cfafdf6e3915fbce50e13d1655425558cb98872c53f802605aa2fb2d569cf"
   - name: preloadImages
     description: |
       Sets the container images to preload to the node that is used for running dockerMachines.

test/e2e/data/infrastructure-docker/v0.3/bases/cluster-with-kcp.yaml

@@ -38,6 +38,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v0.3/bases/md.yaml

@@ -8,6 +8,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"
@@ -24,11 +26,7 @@ spec:
         nodeRegistration:
           criSocket: unix:///var/run/containerd/containerd.sock
           kubeletExtraArgs:
-            # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
-            # This cluster is used in tests where the Kubernetes version is < 1.24
-            cgroup-driver: cgroupfs
             eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
-            fail-swap-on: "false"
 ---
 # MachineDeployment object with
 # - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object

test/e2e/data/infrastructure-docker/v0.4/bases/cluster-with-kcp.yaml

@@ -38,6 +38,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v0.4/bases/md.yaml

@@ -8,6 +8,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v0.4/cluster-template/kustomization.yaml

@@ -1,6 +1,4 @@
 bases:
 - ../bases/cluster-with-kcp.yaml
 - ../bases/md.yaml
-- ../bases/crs.yaml
-- ../bases/mp.yaml
-- ../bases/mhc.yaml
+- ../bases/crs.yaml

test/e2e/data/infrastructure-docker/v1.0/bases/cluster-with-kcp.yaml

@@ -56,6 +56,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml

@@ -8,6 +8,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v1.2/bases/cluster-with-kcp.yaml

@@ -56,6 +56,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v1.2/bases/md.yaml

@@ -8,6 +8,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v1.2/clusterclass-quick-start.yaml

@@ -166,22 +166,6 @@ spec:
         valueFrom:
           template: |
             imageTag: {{ .coreDNSImageTag }}
-  - name: customImage
-    description: "Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments."
-    definitions:
-    - selector:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-        kind: DockerMachineTemplate
-        matchResources:
-          machineDeploymentClass:
-            names:
-            - default-worker
-      jsonPatches:
-      - op: add
-        path: "/spec/template/spec/customImage"
-        valueFrom:
-          template: |
-            kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }}
     - selector:
         apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
         kind: DockerMachineTemplate
@@ -274,6 +258,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e"
       extraMounts:
       - containerPath: "/var/run/docker.sock"
         hostPath: "/var/run/docker.sock"
@@ -285,6 +271,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e"
       extraMounts:
       - containerPath: "/var/run/docker.sock"
         hostPath: "/var/run/docker.sock"

test/e2e/data/infrastructure-docker/v1.3/bases/cluster-with-kcp.yaml

@@ -56,6 +56,8 @@ metadata:
 spec:
   template:
     spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e"
       extraMounts:
         - containerPath: "/var/run/docker.sock"
           hostPath: "/var/run/docker.sock"