Merge pull request #2833 from randomvariable/e2e-upgrade-test

Add optional v1alpha4 and self-hosted management cluster tests
kubernetes-sigs · Oct 8, 2021 · 82f3c98 · 82f3c98
2 parents 1d97ec3 + cc00bcc
commit 82f3c98
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 69 deletions.
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -23,6 +23,7 @@ patchesStrategicMerge:
 - manager_webhook_patch.yaml
 - webhookcainjection_patch.yaml
 - manager_image_patch.yaml
+- manager_probes.yaml
 
 configurations:
   - kustomizeconfig.yaml

diff --git a/config/default/manager_probes.yaml b/config/default/manager_probes.yaml
@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        livenessProbe:
+          failureThreshold: 3
+          periodSeconds: 10
diff --git a/hack/gen-test-flavors.sh b/hack/gen-test-flavors.sh
@@ -23,7 +23,7 @@ root=$(dirname "${BASH_SOURCE[0]}")/..
 kustomize="${root}/hack/tools/bin/kustomize"
 test_dir_path="test/e2e/data/infrastructure-aws"
 test_dir="${root}/${test_dir_path}/"
-src_dir="${test_dir_path}/kustomize_sources/"
+src_dir="${test_dir}/kustomize_sources/"
 generated_dir="${test_dir}/generated"
 
 echo Checking for template sources in "$test_dir"

diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml
@@ -47,21 +47,17 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
         contract: v1alpha4
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/core-components.yaml"
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/core-components.yaml"
         type: "url"
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -73,8 +69,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
   - name: kubeadm
@@ -91,17 +85,15 @@ providers:
         replacements:
           - old: --metrics-addr=127.0.0.1:8080
             new: --metrics-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/bootstrap-components.yaml"
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/bootstrap-components.yaml"
         type: "url"
         contract: v1alpha4
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -113,8 +105,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
 
@@ -132,18 +122,16 @@ providers:
         replacements:
           - old: --metrics-addr=127.0.0.1:8080
             new: --metrics-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
         # Use manifest from source files
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/control-plane-components.yaml"
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/control-plane-components.yaml"
         type: "url"
         contract: v1alpha4
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -155,8 +143,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
   - name: aws
@@ -181,26 +167,27 @@ providers:
         value: ../../../config/default
         contract: v1beta1
         files:
-          - sourcePath: "./shared/v1beta1_provider/metadata.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-capi-upgrade.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-ssm.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-multi-az.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-limit-az.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-spot-instances.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-md-remediation.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-external-cloud-provider.yaml"
           - sourcePath: "./infrastructure-aws/generated/cluster-template-kcp-remediation.yaml"
           - sourcePath: "./infrastructure-aws/generated/cluster-template-kcp-scale-in.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-limit-az.yaml"
           - sourcePath: "./infrastructure-aws/generated/cluster-template-machine-pool.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-upgrade-to-main.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-simple-multitenancy.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-md-remediation.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-multi-az.yaml"
           - sourcePath: "./infrastructure-aws/generated/cluster-template-nested-multitenancy.yaml"
-          - sourcePath: "./infrastructure-aws/generated/cluster-template-external-cloud-provider.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-remote-management-cluster.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-simple-multitenancy.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-spot-instances.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-ssm.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template-upgrade-to-main.yaml"
+          - sourcePath: "./infrastructure-aws/generated/cluster-template.yaml"
+          - sourcePath: "./shared/v1beta1_provider/metadata.yaml"
         replacements:
+          # To allow bugs to be catched.
+          - old: "failureThreshold: 3"
+            new: "failureThreshold: 12"
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
           - old: gcr.io/k8s-staging-cluster-api/cluster-api-aws-controller-amd64:dev
@@ -227,10 +214,10 @@ variables:
   MULTI_TENANCY_ROLE_NAME: "multi-tenancy-role"
   MULTI_TENANCY_NESTED_ROLE_NAME: "multi-tenancy-nested-role"
   IP_FAMILY: "IPv4"
-  # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
+  INIT_WITH_BINARY_V1ALPHA3: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
+  INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
+    # INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
   # the management cluster to be upgraded.
-  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
-  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
   INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
 
 intervals:
@@ -239,7 +226,7 @@ intervals:
   default/wait-worker-nodes: ["10m", "10s"]
   conformance/wait-control-plane: ["30m", "10s"]
   conformance/wait-worker-nodes: ["30m", "10s"]
-  default/wait-controllers: ["3m", "10s"]
+  default/wait-controllers: ["5m", "10s"]
   default/wait-delete-cluster: ["20m", "10s"]
   default/wait-machine-upgrade: ["30m", "10s"]
   default/wait-machine-status: ["20m", "10s"]

diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml
@@ -45,21 +45,17 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
         contract: v1alpha4
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/core-components.yaml"
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/core-components.yaml"
         type: "url"
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -71,8 +67,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
   - name: kubeadm
@@ -89,17 +83,15 @@ providers:
         replacements:
           - old: --metrics-addr=127.0.0.1:8080
             new: --metrics-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/bootstrap-components.yaml"
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/bootstrap-components.yaml"
         type: "url"
         contract: v1alpha4
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -111,8 +103,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
   - name: kubeadm
@@ -129,18 +119,16 @@ providers:
         replacements:
           - old: --metrics-addr=127.0.0.1:8080
             new: --metrics-addr=:8080
-      - name: v0.4.3 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+      - name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
         # Use manifest from source files
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.3/control-plane-components.yaml"
+        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/control-plane-components.yaml"
         type: "url"
         contract: v1alpha4
         files:
           - sourcePath: "./shared/v1alpha4/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
       - name: v1.0.0
@@ -152,8 +140,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
   - name: aws
@@ -167,8 +153,6 @@ providers:
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
           - old: --metrics-bind-addr=127.0.0.1:8080
             new: --metrics-bind-addr=:8080
           - old: gcr.io/k8s-staging-cluster-api/cluster-api-aws-controller-amd64:dev
@@ -204,7 +188,7 @@ intervals:
   default/wait-cluster: ["30m", "10s"]
   default/wait-control-plane: ["30m", "10s"]
   default/wait-worker-nodes: ["30m", "10s"]
-  default/wait-controllers: ["3m", "10s"]
+  default/wait-controllers: ["5m", "10s"]
   default/wait-delete-cluster: ["35m", "30s"]
   default/wait-delete-machine: ["10m", "10s"]
   default/wait-delete-machine-deployment: ["10m", "10s"]

diff --git a/test/e2e/data/infrastructure-aws/capi-upgrades/v1alpha3/cluster-template.yaml b/test/e2e/data/infrastructure-aws/capi-upgrades/v1alpha3/cluster-template.yaml
@@ -21,6 +21,9 @@ kind: AWSCluster
 metadata:
   name: "${CLUSTER_NAME}"
 spec:
+  networkSpec:
+    vpc:
+      availabilityZoneUsageLimit: 1
   region: "${AWS_REGION}"
   sshKeyName: "${AWS_SSH_KEY_NAME}"
 ---

diff --git a/test/e2e/data/infrastructure-aws/capi-upgrades/v1alpha4/cluster-template.yaml b/test/e2e/data/infrastructure-aws/capi-upgrades/v1alpha4/cluster-template.yaml
@@ -23,6 +23,9 @@ kind: AWSCluster
 metadata:
   name: ${CLUSTER_NAME}
 spec:
+  network:
+    vpc:
+      availabilityZoneUsageLimit: 1
   region: ${AWS_REGION}
   sshKeyName: ${AWS_SSH_KEY_NAME}
 ---

diff --git a/...e_sources/capi-upgrade/kustomization.yaml → ...ote-management-cluster/kustomization.yaml b/...e_sources/capi-upgrade/kustomization.yaml → ...ote-management-cluster/kustomization.yaml
@@ -1,4 +1,4 @@
 resources:
-  - ../default
+  - ../limit-az
 patchesStrategicMerge:
   - patches/image-injection.yaml
diff --git a/...capi-upgrade/patches/image-injection.yaml → ...ment-cluster/patches/image-injection.yaml b/...capi-upgrade/patches/image-injection.yaml → ...ment-cluster/patches/image-injection.yaml
diff --git a/test/e2e/shared/aws.go b/test/e2e/shared/aws.go
@@ -337,7 +337,7 @@ func ensureTestImageUploaded(e2eCtx *E2EContext) (string, string, error) {
 		return "", "", err
 	}
 
-	Byf("Image uploaded to s3:///%s/%s", bucketName, imageSha)
+	Byf("Image uploaded to s3://%s/%s", bucketName, imageSha)
 	return bucketName, imageSha, nil
 }