✨ Update kube-vip version and add e2e tests

Makefile

@@ -211,6 +211,7 @@ e2e-test-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1al
 e2e-test-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta1
 	mkdir -p $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/
 	$(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kube-vip.yaml
 	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kcp-scale-in.yaml
 	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-node-drain --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-node-drain.yaml
 	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-md-remediation.yaml
@@ -277,6 +278,7 @@ generate: ## Generate code
 
 .PHONY: generate-templates
 generate-templates: $(KUSTOMIZE) ## Generate cluster templates
+	$(KUSTOMIZE) build templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip-crs-cni.yaml
 	$(KUSTOMIZE) build templates/experimental-kube-vip --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip.yaml
 	$(KUSTOMIZE) build templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-crs-cni.yaml
 	$(KUSTOMIZE) build templates/addons/calico > templates/addons/calico.yaml

templates/addons/calico.yaml

@@ -2497,7 +2497,7 @@ spec:
           value: node
         - name: DATASTORE_TYPE
           value: kubernetes
-        image: quay.io/calico/kube-controllers:v3.20.5
+        image: quay.io/calico/kube-controllers:v3.20.6
         livenessProbe:
           exec:
             command:
@@ -2603,7 +2603,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/node:v3.20.5
+        image: quay.io/calico/node:v3.20.6
         lifecycle:
           preStop:
             exec:
@@ -2677,7 +2677,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/cni:v3.20.5
+        image: quay.io/calico/cni:v3.20.6
         name: upgrade-ipam
         securityContext:
           privileged: true
@@ -2711,7 +2711,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/cni:v3.20.5
+        image: quay.io/calico/cni:v3.20.6
         name: install-cni
         securityContext:
           privileged: true
@@ -2720,7 +2720,7 @@ spec:
           name: cni-bin-dir
         - mountPath: /host/etc/cni/net.d
           name: cni-net-dir
-      - image: quay.io/calico/pod2daemon-flexvol:v3.20.5
+      - image: quay.io/calico/pod2daemon-flexvol:v3.20.6
         name: flexvol-driver
         securityContext:
           privileged: true

templates/cluster-template-crs-cni.yaml

@@ -146,7 +146,7 @@ spec:
       systemctl restart networking
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
         export KUBECONFIG=/etc/kubernetes/admin.conf
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.4.3/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

templates/cluster-template-kube-vip-crs-cni.yaml

@@ -0,0 +1,231 @@
+apiVersion: v1
+data: ${CNI_RESOURCES}
+kind: ConfigMap
+metadata:
+  name: ${CLUSTER_NAME}-crs-cni
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: ${CLUSTER_NAME}-crs-cni
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: ${CLUSTER_NAME}-crs-cni
+  resources:
+  - kind: ConfigMap
+    name: ${CLUSTER_NAME}-crs-cni
+  strategy: ApplyOnce
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+            cloud-provider: external
+            provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+      preKubeadmCommands:
+      - |-
+        sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+        swapoff -a
+        mount -a
+        cat <<EOF > /etc/modules-load.d/containerd.conf
+        overlay
+        br_netfilter
+        EOF
+        modprobe overlay
+        modprobe br_netfilter
+        cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
+        net.bridge.bridge-nf-call-iptables  = 1
+        net.ipv4.ip_forward                 = 1
+        net.bridge.bridge-nf-call-ip6tables = 1
+        EOF
+        sysctl --system
+        apt-get -y update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+        curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+        echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+        apt-get update -y
+        TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+        RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
+        DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  labels:
+    cni: ${CLUSTER_NAME}-crs-cni
+  name: ${CLUSTER_NAME}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - ${POD_CIDR:=192.168.0.0/16}
+    services:
+      cidrBlocks:
+      - ${SERVICE_CIDR:=172.26.0.0/16}
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KubeadmControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: PacketCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+    pool: worker-a
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+      pool: worker-a
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+        pool: worker-a
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-worker-a
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: PacketMachineTemplate
+        name: ${CLUSTER_NAME}-worker-a
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      apiServer:
+        extraArgs:
+          cloud-provider: external
+      controllerManager:
+        extraArgs:
+          cloud-provider: external
+    initConfiguration:
+      nodeRegistration:
+        kubeletExtraArgs:
+          cloud-provider: external
+          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+    joinConfiguration:
+      nodeRegistration:
+        ignorePreflightErrors:
+        - DirAvailable--etc-kubernetes-manifests
+        kubeletExtraArgs:
+          cloud-provider: external
+          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+    postKubeadmCommands:
+    - |-
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        export KUBECONFIG=/etc/kubernetes/admin.conf
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
+        kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
+        kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      fi
+    preKubeadmCommands:
+    - |
+      sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+      swapoff -a
+      mount -a
+      cat <<EOF > /etc/modules-load.d/containerd.conf
+      overlay
+      br_netfilter
+      EOF
+      modprobe overlay
+      modprobe br_netfilter
+      cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.ipv4.ip_forward                 = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+      EOF
+      sysctl --system
+      apt-get -y update
+      DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+      curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+      echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+      apt-get update -y
+      TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+      RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
+      DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+      done
+      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+      --interface "lo" \
+      --vip "{{ .controlPlaneEndpoint }}" \
+      --controlplane \
+      --bgp \
+      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+      rm /run/metadata.json
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: PacketMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  facility: ${FACILITY}
+  projectID: ${PROJECT_ID}
+  vipManager: KUBE_VIP
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      billingCycle: hourly
+      machineType: ${CONTROLPLANE_NODE_TYPE}
+      os: ${NODE_OS:=ubuntu_18_04}
+      sshKeys:
+      - ${SSH_KEY}
+      tags: []
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  template:
+    spec:
+      billingCycle: hourly
+      machineType: ${WORKER_NODE_TYPE}
+      os: ${NODE_OS:=ubuntu_18_04}
+      sshKeys:
+      - ${SSH_KEY}
+      tags: []

templates/cluster-template-kube-vip.yaml

@@ -39,8 +39,6 @@ spec:
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
-  labels:
-    cni: ${CLUSTER_NAME}-kube-vip
   name: ${CLUSTER_NAME}
 spec:
   clusterNetwork:
@@ -120,7 +118,7 @@ spec:
     - |-
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
         export KUBECONFIG=/etc/kubernetes/admin.conf
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.4.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
@@ -154,7 +152,7 @@ spec:
       for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
         ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
       done
-      KVVERSION="${KUBE_VIP_VERSION:=v0.4.2}"
+      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
       ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
       ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
       --interface "lo" \

templates/cluster-template.yaml

@@ -67,7 +67,7 @@ spec:
         systemctl restart networking
         if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
           export KUBECONFIG=/etc/kubernetes/admin.conf
-          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.4.3/deployment.yaml
+          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
           export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
           kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
           kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

templates/experimental-kube-vip-crs-cni/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../experimental-kube-vip
+- ../bases/crs-cni.yaml
+patches:
+- patch: |-
+    kind: Cluster
+    apiVersion: cluster.x-k8s.io/v1beta1
+    metadata:
+      name: not-used
+      labels:
+        cni: "${CLUSTER_NAME}-crs-cni"
+  target:
+    kind: Cluster

templates/experimental-kube-vip/kustomization.yaml

@@ -4,15 +4,6 @@ kind: Kustomization
 resources:
 - ../cluster-template.yaml
 patches:
-- patch: |-
-    kind: Cluster
-    apiVersion: cluster.x-k8s.io/v1beta1
-    metadata:
-      name: not-used
-      labels:
-        cni: "${CLUSTER_NAME}-kube-vip"
-  target:
-    kind: Cluster
 - patch: |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: PacketCluster
@@ -56,7 +47,7 @@ patches:
             for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
               ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
             done
-            KVVERSION="${KUBE_VIP_VERSION:=v0.4.2}"
+            KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
             ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
             ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
             --interface "lo" \
@@ -72,7 +63,7 @@ patches:
           - |
             if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
               export KUBECONFIG=/etc/kubernetes/admin.conf
-              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.4.0/deployment.yaml
+              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
               export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
               kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
               kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

test/e2e/capi_e2e_clusterctl_upgrade_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 /*

test/e2e/capi_e2e_conformance_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 /*