Skip to content

Commit

Permalink
Add e2e test to reproduce SSA apiVersion issue
Browse files Browse the repository at this point in the history
Signed-off-by: Stefan Büringer buringerst@vmware.com
  • Loading branch information
sbueringer committed Feb 13, 2024
1 parent 6249008 commit d30cb88
Show file tree
Hide file tree
Showing 12 changed files with 755 additions and 209 deletions.
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -562,11 +562,15 @@ generate-doctoc:
TRACE=$(TRACE) ./hack/generate-doctoc.sh

.PHONY: generate-e2e-templates
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions

DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker
INMEMORY_TEMPLATES := test/e2e/data/infrastructure-inmemory

.PHONY: generate-e2e-templates-v0.4
generate-e2e-templates-v0.4: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml

.PHONY: generate-e2e-templates-v1.0
generate-e2e-templates-v1.0: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.0/cluster-template.yaml
Expand Down
557 changes: 383 additions & 174 deletions test/e2e/clusterctl_upgrade.go

Large diffs are not rendered by default.

56 changes: 56 additions & 0 deletions test/e2e/clusterctl_upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/utils/ptr"

clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
)

var (
Expand All @@ -34,6 +36,60 @@ var (
providerDockerPrefix = "docker:v%s"
)

var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func() {
// Get v0.4 latest stable release
version04 := "0.4"
stableRelease04, err := GetStableReleaseOfMinor(ctx, version04)
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version04)

// Get v1.6 latest stable release
version16 := "1.6"
stableRelease16, err := GetStableReleaseOfMinor(ctx, version16)
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version16)

ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InfrastructureProvider: ptr.To("docker"),
// Configuration for the initial provider deployment.
InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease04),
// We have to pin the providers because with `InitWithProvidersContract` the test would
// use the latest version for the contract (which is v1.3.X for v1beta1).
InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease04),
InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
InitWithControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease04)},
// We have to set this to an empty array as clusterctl v0.4 doesn't support
// runtime extension providers. If we don't do this the test will automatically
// try to deploy the latest version of our test-extension from docker.yaml.
InitWithRuntimeExtensionProviders: []string{},
// Configuration for the provider upgrades.
Upgrades: []ClusterctlUpgradeSpecInputUpgrade{
{ // Upgrade to 1.6.
WithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease16),
CoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease16),
BootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
ControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
InfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease16)},
},
{ // Upgrade to latest v1beta1.
Contract: clusterv1.GroupVersion.Version,
},
},
// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.4/bases.
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
InitWithKubernetesVersion: "v1.23.17",
WorkloadKubernetesVersion: "v1.23.17",
MgmtFlavor: "topology",
WorkloadFlavor: "",
}
})
})

var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
// Get v1.0 latest stable release
version := "1.0"
Expand Down
37 changes: 37 additions & 0 deletions test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ providers:
- name: cluster-api
type: CoreProvider
versions:
- name: "{go://sigs.k8s.io/cluster-api@v0.4}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v0.4}/core-components.yaml"
type: "url"
contract: v1alpha4
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.0}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.0}/core-components.yaml"
type: "url"
Expand Down Expand Up @@ -73,6 +82,15 @@ providers:
- name: kubeadm
type: BootstrapProvider
versions:
- name: "{go://sigs.k8s.io/cluster-api@v0.4}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v0.4}/bootstrap-components.yaml"
type: "url"
contract: v1alpha4
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.0}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.0}/bootstrap-components.yaml"
type: "url"
Expand Down Expand Up @@ -111,6 +129,15 @@ providers:
- name: kubeadm
type: ControlPlaneProvider
versions:
- name: "{go://sigs.k8s.io/cluster-api@v0.4}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v0.4}/control-plane-components.yaml"
type: "url"
contract: v1alpha4
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.0}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.0}/control-plane-components.yaml"
type: "url"
Expand Down Expand Up @@ -149,6 +176,16 @@ providers:
- name: docker
type: InfrastructureProvider
versions:
- name: "{go://sigs.k8s.io/cluster-api@v0.4}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v0.4}/infrastructure-components-development.yaml"
type: "url"
contract: v1alpha4
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.0}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.0}/infrastructure-components-development.yaml"
type: "url"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
---
# DockerCluster object referenced by the Cluster object
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: DockerCluster
metadata:
name: '${CLUSTER_NAME}'
---
# Cluster object with
# - Reference to the KubeadmControlPlane object
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
apiVersion: cluster.x-k8s.io/v1alpha4
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: DockerCluster
name: '${CLUSTER_NAME}'
controlPlaneRef:
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
name: "${CLUSTER_NAME}-control-plane"
---
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec:
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
metadata:
name: "${CLUSTER_NAME}-control-plane"
labels:
kcp-adoption.step2: ""
spec:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
machineTemplate:
infrastructureRef:
kind: DockerMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
name: "${CLUSTER_NAME}-control-plane"
kubeadmConfigSpec:
clusterConfiguration:
controllerManager:
extraArgs: {enable-hostpath-provisioner: 'true'}
apiServer:
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
initConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
# This cluster is used in tests where the Kubernetes version is < 1.24
cgroup-driver: cgroupfs
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
fail-swap-on: "false"
joinConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
# This cluster is used in tests where the Kubernetes version is < 1.24
cgroup-driver: cgroupfs
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
fail-swap-on: "false"
version: "${KUBERNETES_VERSION}"
24 changes: 24 additions & 0 deletions test/e2e/data/infrastructure-docker/v0.4/bases/crs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
# ConfigMap object referenced by the ClusterResourceSet object and with
# the CNI resource defined in the test config file
apiVersion: v1
kind: ConfigMap
metadata:
name: "cni-${CLUSTER_NAME}-crs-0"
data: ${CNI_RESOURCES}
binaryData:
---
# ClusterResourceSet object with
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
apiVersion: addons.cluster.x-k8s.io/v1alpha4
kind: ClusterResourceSet
metadata:
name: "${CLUSTER_NAME}-crs-0"
spec:
strategy: ApplyOnce
clusterSelector:
matchLabels:
cni: "${CLUSTER_NAME}-crs-0"
resources:
- name: "cni-${CLUSTER_NAME}-crs-0"
kind: ConfigMap
57 changes: 57 additions & 0 deletions test/e2e/data/infrastructure-docker/v0.4/bases/md.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
---
# DockerMachineTemplate referenced by the MachineDeployment and with
# - extraMounts for the docker sock, thus allowing self-hosting test
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmConfigTemplate referenced by the MachineDeployment
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
kind: KubeadmConfigTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
# This cluster is used in tests where the Kubernetes version is < 1.24
cgroup-driver: cgroupfs
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
fail-swap-on: "false"
---
# MachineDeployment object
apiVersion: cluster.x-k8s.io/v1alpha4
kind: MachineDeployment
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
clusterName: "${CLUSTER_NAME}"
replicas: ${WORKER_MACHINE_COUNT}
selector:
matchLabels:
template:
spec:
clusterName: "${CLUSTER_NAME}"
version: "${KUBERNETES_VERSION}"
bootstrap:
configRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
kind: KubeadmConfigTemplate
infrastructureRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: DockerMachineTemplate
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
bases:
- ../bases/cluster-with-kcp.yaml
- ../bases/md.yaml
- ../bases/crs.yaml
12 changes: 12 additions & 0 deletions test/e2e/data/shared/v0.4/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
kind: Metadata
releaseSeries:
- major: 0
minor: 4
contract: v1alpha4
- major: 0
minor: 3
contract: v1alpha3
- major: 0
minor: 2
contract: v1alpha2
Loading

0 comments on commit d30cb88

Please sign in to comment.