From 2b02c696981f1079e4324f4dedc2eb8c91780f3c Mon Sep 17 00:00:00 2001
From: Oscar Cobles <oscar.cobles@eu.equinix.com>
Date: Thu, 8 Sep 2022 15:33:53 +0200
Subject: [PATCH 01/16] Update go install script and add kustomize install
 script

---
 Makefile              | 10 ++++++++--
 scripts/go_install.sh | 13 +------------
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index 816c51efd..ab399bbbd 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,7 @@ GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)
 
 # Keep at 4.0.4 until we figure out how to get later verisons to not mangle the calico yamls
+# HACK bump latest version once https://github.com/kubernetes-sigs/kustomize/issues/947 is fixed
 KUSTOMIZE_VER := v4.0.4
 KUSTOMIZE_BIN := kustomize
 KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)
@@ -224,8 +225,13 @@ $(ENVSUBST): ## Build envsubst from tools folder.
 $(GOLANGCI_LINT): ## Build golangci-lint from tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)
 
-$(KUSTOMIZE): ## Build kustomize from tools folder.
-	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kustomize/kustomize/v4 $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER)
+## HACK replace with $(GO_INSTALL) once https://github.com/kubernetes-sigs/kustomize/issues/947 is fixed
+$(KUSTOMIZE): ## Put kustomize into tools folder.
+	mkdir -p $(TOOLS_BIN_DIR)
+	rm -f $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)*
+	curl -fsSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- $(KUSTOMIZE_VER:v%=%) $(TOOLS_BIN_DIR)
+	mv "$(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)" $(KUSTOMIZE)
+	ln -sf $(KUSTOMIZE) "$(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)"
 
 $(CONTROLLER_GEN): ## Build controller-gen from tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER)
diff --git a/scripts/go_install.sh b/scripts/go_install.sh
index 224f8ae2d..1e562ea26 100755
--- a/scripts/go_install.sh
+++ b/scripts/go_install.sh
@@ -37,20 +37,9 @@ if [ -z "${GOBIN}" ]; then
   exit 1
 fi
 
-tmp_dir=$(mktemp -d -t goinstall_XXXXXXXXXX)
-function clean {
-  rm -rf "${tmp_dir}"
-}
-trap clean EXIT
-
 rm "${GOBIN}/${2}"* || true
 
-cd "${tmp_dir}"
-
-# create a new module in the tmp directory
-go mod init fake/mod
-
 # install the golang module specified as the first argument
-go get -tags tools "${1}@${3}"
+go install -tags tools "${1}@${3}"
 mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}"
 ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}"

From b2dbab8f6f36dab75f830bbf3e35df0e11c5fc67 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Fri, 9 Sep 2022 14:28:22 -0500
Subject: [PATCH 02/16] Remove docker.io from apt-get remove line Install
 containerd.io instead of containerd Add quotes around the kubernetes pause
 image

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template.yaml                    | 12 ++++++------
 templates/experimental-kube-vip/kustomization.yaml | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
index ee9625bb4..81984c0ac 100644
--- a/templates/cluster-template.yaml
+++ b/templates/cluster-template.yaml
@@ -49,7 +49,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
         if [[ "$major_vers" -ge 20 ]]; then
@@ -63,14 +63,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         containerd config default > /etc/containerd/config.toml
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
         ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
     postKubeadmCommands:
@@ -215,7 +215,7 @@ spec:
           sysctl --system
           export DEBIAN_FRONTEND=noninteractive
           apt-get update -y
-          apt-get remove -y docker docker-engine docker.io containerd runc
+          apt-get remove -y docker docker-engine containerd runc
           apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
           mkdir -p /etc/apt/keyrings
           curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -225,12 +225,12 @@ spec:
           apt-get update -y
           TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
           RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-          apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+          apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
           cat  <<EOF > /etc/crictl.yaml
           runtime-endpoint: unix:///run/containerd/containerd.sock
           image-endpoint: unix:///run/containerd/containerd.sock
           EOF
           containerd config default > /etc/containerd/config.toml
           sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-          sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+          sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
           systemctl restart containerd
\ No newline at end of file
diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 2535beec2..4cf701e69 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -37,7 +37,7 @@ patches:
             sysctl --system
             export DEBIAN_FRONTEND=noninteractive
             apt-get update -y
-            apt-get remove -y docker docker-engine docker.io containerd runc
+            apt-get remove -y docker docker-engine containerd runc
             apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
             mkdir -p /etc/apt/keyrings
             curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -47,14 +47,14 @@ patches:
             apt-get update -y
             TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
             RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-            apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+            apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
             cat  <<EOF > /etc/crictl.yaml
             runtime-endpoint: unix:///run/containerd/containerd.sock
             image-endpoint: unix:///run/containerd/containerd.sock
             EOF
             containerd config default > /etc/containerd/config.toml
             sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-            sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+            sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
             ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
@@ -113,7 +113,7 @@ patches:
               sysctl --system
               export DEBIAN_FRONTEND=noninteractive
               apt-get update -y
-              apt-get remove -y docker docker-engine docker.io containerd runc
+              apt-get remove -y docker docker-engine containerd runc
               apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
               mkdir -p /etc/apt/keyrings
               curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -123,12 +123,12 @@ patches:
               apt-get update -y
               TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
               RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-              apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+              apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
               cat  <<EOF > /etc/crictl.yaml
               runtime-endpoint: unix:///run/containerd/containerd.sock
               image-endpoint: unix:///run/containerd/containerd.sock
               EOF
               containerd config default > /etc/containerd/config.toml
               sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-              sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+              sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
               systemctl restart containerd
\ No newline at end of file

From 0477692b2a2bf515cbc1493a810549461a6712d3 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Fri, 9 Sep 2022 15:44:04 -0500
Subject: [PATCH 03/16] ran make generate

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template-crs-cni.yaml          | 12 ++++++------
 templates/cluster-template-kube-vip-crs-cni.yaml | 12 ++++++------
 templates/cluster-template-kube-vip.yaml         | 12 ++++++------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/templates/cluster-template-crs-cni.yaml b/templates/cluster-template-crs-cni.yaml
index c6f44eb6e..83e288e8c 100644
--- a/templates/cluster-template-crs-cni.yaml
+++ b/templates/cluster-template-crs-cni.yaml
@@ -48,7 +48,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -58,14 +58,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -187,7 +187,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
       if [[ "$major_vers" -ge 20 ]]; then
@@ -201,14 +201,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       containerd config default > /etc/containerd/config.toml
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
   machineTemplate:
diff --git a/templates/cluster-template-kube-vip-crs-cni.yaml b/templates/cluster-template-kube-vip-crs-cni.yaml
index 4d3cd9a78..2995a60fa 100644
--- a/templates/cluster-template-kube-vip-crs-cni.yaml
+++ b/templates/cluster-template-kube-vip-crs-cni.yaml
@@ -48,7 +48,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -58,14 +58,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -180,7 +180,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       mkdir -p /etc/apt/keyrings
       curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -190,14 +190,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       containerd config default > /etc/containerd/config.toml
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
       curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
diff --git a/templates/cluster-template-kube-vip.yaml b/templates/cluster-template-kube-vip.yaml
index 244c30a08..7cf3d9f4b 100644
--- a/templates/cluster-template-kube-vip.yaml
+++ b/templates/cluster-template-kube-vip.yaml
@@ -29,7 +29,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -39,14 +39,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -159,7 +159,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       mkdir -p /etc/apt/keyrings
       curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -169,14 +169,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       containerd config default > /etc/containerd/config.toml
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
       curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata

From f29d64a315eb0c082006fcd3061173421aad04ca Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:15:52 -0500
Subject: [PATCH 04/16] Update defaults list

Removed things that were required and had no defaults
Added a variable for CPEM verison

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 README.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 8df27d63d..5754c68a6 100644
--- a/README.md
+++ b/README.md
@@ -34,13 +34,11 @@ You should then follow the [Cluster API Quick Start Guide](https://cluster-api.s
 
 If you do not change the generated `yaml` files, it will use defaults. You can look in the [templates/cluster-template.yaml](./templates/cluster-template.yaml) file for details.
 
-* `CLUSTER_NAME`                 (defaults to `my-cluster`)
-* `CONTROL_PLANE_MACHINE_COUNT`  (defaults to `1`)
+* `CPEM_VERSION`                 (defaults to `v3.5.0`)
 * `KUBE_VIP_VERSION`             (defaults to `v0.5.0`)
 * `NODE_OS`                      (defaults to `ubuntu_18_04`)
 * `POD_CIDR`                     (defaults to `192.168.0.0/16`)
 * `SERVICE_CIDR`                 (defaults to `172.26.0.0/16`)
-* `WORKER_MACHINE_COUNT`         (defaults to `0`)
   
 ### Reserved Hardware
 

From e034b22e2dc548609d7aaec097baab1483ff7073 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:22:01 -0500
Subject: [PATCH 05/16] Fix environment setup

Move the commands to run on all control plane nodes
Remove unnecessary KUBECONFIG export to save a line

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template.yaml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
index 81984c0ac..85ae09384 100644
--- a/templates/cluster-template.yaml
+++ b/templates/cluster-template.yaml
@@ -82,13 +82,12 @@ spec:
           netmask 255.255.255.255
         EOF
         systemctl restart networking
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
         if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-          export KUBECONFIG=/etc/kubernetes/admin.conf
-          mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-          echo "source <(kubectl completion bash)" >> /root/.bashrc
-          echo "alias k=kubectl" >> /root/.bashrc
-          echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
           export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
           kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
           kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

From 41f91dada32cfc2e8777ab923565917e445c4909 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:23:29 -0500
Subject: [PATCH 06/16] Fix kube-vip setup

Move the kube-vip manifest setup to after kubeadm is run,
this avoids a situation where kube-vip fights with previous
control plane nodes before they've joined the cluster.
Move the crictl/kubectl commands to run on all control plane nodes
Remove unnecessary KUBECONFIG export to save a line

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 .../experimental-kube-vip/kustomization.yaml      | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 4cf701e69..51dafac9b 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -57,6 +57,8 @@ patches:
             sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
             ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+        postKubeadmCommands:
+          - |
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
             for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
               ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
@@ -73,15 +75,12 @@ patches:
             --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
             --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
             rm /run/metadata.json
-        postKubeadmCommands:
-          - |
+            mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+            echo "source <(kubectl completion bash)" >> /root/.bashrc
+            echo "alias k=kubectl" >> /root/.bashrc
+            echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-              export KUBECONFIG=/etc/kubernetes/admin.conf
-              mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-              echo "source <(kubectl completion bash)" >> /root/.bashrc
-              echo "alias k=kubectl" >> /root/.bashrc
-              echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
               export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
               kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
               kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

From e585a5468e4c5102a5caecde3f31b1637f93d5d2 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Tue, 13 Sep 2022 14:11:57 -0500
Subject: [PATCH 07/16] Only setup kube-vip before kubeadm on first node

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 .../experimental-kube-vip/kustomization.yaml  | 53 +++++++++++++------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 51dafac9b..d0bd53955 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -56,34 +56,53 @@ patches:
             sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
             sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
-            ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+            if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+              ip addr add {{ .controlPlaneEndpoint }} dev lo
+              curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+              for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+                ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+              done
+              KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+              ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+              ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+              --interface "lo" \
+              --vip "{{ .controlPlaneEndpoint }}" \
+              --controlplane \
+              --bgp \
+              --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+              --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+              --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+              --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+              rm /run/metadata.json
+            fi
         postKubeadmCommands:
           - |
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
             for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
               ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
             done
-            KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-            ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-            ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-            --interface "lo" \
-            --vip "{{ .controlPlaneEndpoint }}" \
-            --controlplane \
-            --bgp \
-            --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-            --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-            --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-            --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-            rm /run/metadata.json
-            mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
-            echo "source <(kubectl completion bash)" >> /root/.bashrc
-            echo "alias k=kubectl" >> /root/.bashrc
-            echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
               export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
               export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
               kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
               kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+            else
+              KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+              ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+              ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+              --interface "lo" \
+              --vip "{{ .controlPlaneEndpoint }}" \
+              --controlplane \
+              --bgp \
+              --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+              --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+              --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+              --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+              rm /run/metadata.json
+              mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+              echo "source <(kubectl completion bash)" >> /root/.bashrc
+              echo "alias k=kubectl" >> /root/.bashrc
+              echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             fi
 - patch: |
     kind: KubeadmConfigTemplate

From d5fa66a054c876f4fe658703c650bccb054e1334 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Tue, 13 Sep 2022 14:12:10 -0500
Subject: [PATCH 08/16] Run make generate

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template-crs-cni.yaml       | 11 ++--
 .../cluster-template-kube-vip-crs-cni.yaml    | 64 ++++++++++++-------
 templates/cluster-template-kube-vip.yaml      | 64 ++++++++++++-------
 3 files changed, 87 insertions(+), 52 deletions(-)

diff --git a/templates/cluster-template-crs-cni.yaml b/templates/cluster-template-crs-cni.yaml
index 83e288e8c..9500abcb3 100644
--- a/templates/cluster-template-crs-cni.yaml
+++ b/templates/cluster-template-crs-cni.yaml
@@ -157,13 +157,12 @@ spec:
         netmask 255.255.255.255
       EOF
       systemctl restart networking
+      mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+      echo "source <(kubectl completion bash)" >> /root/.bashrc
+      echo "alias k=kubectl" >> /root/.bashrc
+      echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
diff --git a/templates/cluster-template-kube-vip-crs-cni.yaml b/templates/cluster-template-kube-vip-crs-cni.yaml
index 2995a60fa..541bcd100 100644
--- a/templates/cluster-template-kube-vip-crs-cni.yaml
+++ b/templates/cluster-template-kube-vip-crs-cni.yaml
@@ -150,16 +150,32 @@ spec:
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
     postKubeadmCommands:
     - |-
+      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+      done
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      else
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       fi
     preKubeadmCommands:
     - |
@@ -199,23 +215,25 @@ spec:
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
       sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
-      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
-      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
-      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
-        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
-      done
-      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-      --interface "lo" \
-      --vip "{{ .controlPlaneEndpoint }}" \
-      --controlplane \
-      --bgp \
-      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-      rm /run/metadata.json
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        ip addr add {{ .controlPlaneEndpoint }} dev lo
+        curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+        for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+          ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+        done
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+      fi
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
diff --git a/templates/cluster-template-kube-vip.yaml b/templates/cluster-template-kube-vip.yaml
index 7cf3d9f4b..832f45340 100644
--- a/templates/cluster-template-kube-vip.yaml
+++ b/templates/cluster-template-kube-vip.yaml
@@ -129,16 +129,32 @@ spec:
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
     postKubeadmCommands:
     - |-
+      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+      done
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      else
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       fi
     preKubeadmCommands:
     - |
@@ -178,23 +194,25 @@ spec:
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
       sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
-      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
-      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
-      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
-        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
-      done
-      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-      --interface "lo" \
-      --vip "{{ .controlPlaneEndpoint }}" \
-      --controlplane \
-      --bgp \
-      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-      rm /run/metadata.json
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        ip addr add {{ .controlPlaneEndpoint }} dev lo
+        curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+        for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+          ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+        done
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+      fi
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

From 9872228ac41f1bbc1f28eb2742511d9e4656019c Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Fri, 9 Sep 2022 14:28:22 -0500
Subject: [PATCH 09/16] Remove docker.io from apt-get remove line Install
 containerd.io instead of containerd Add quotes around the kubernetes pause
 image

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template.yaml                    | 12 ++++++------
 templates/experimental-kube-vip/kustomization.yaml | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
index ee9625bb4..81984c0ac 100644
--- a/templates/cluster-template.yaml
+++ b/templates/cluster-template.yaml
@@ -49,7 +49,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
         if [[ "$major_vers" -ge 20 ]]; then
@@ -63,14 +63,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         containerd config default > /etc/containerd/config.toml
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
         ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
     postKubeadmCommands:
@@ -215,7 +215,7 @@ spec:
           sysctl --system
           export DEBIAN_FRONTEND=noninteractive
           apt-get update -y
-          apt-get remove -y docker docker-engine docker.io containerd runc
+          apt-get remove -y docker docker-engine containerd runc
           apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
           mkdir -p /etc/apt/keyrings
           curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -225,12 +225,12 @@ spec:
           apt-get update -y
           TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
           RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-          apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+          apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
           cat  <<EOF > /etc/crictl.yaml
           runtime-endpoint: unix:///run/containerd/containerd.sock
           image-endpoint: unix:///run/containerd/containerd.sock
           EOF
           containerd config default > /etc/containerd/config.toml
           sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-          sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+          sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
           systemctl restart containerd
\ No newline at end of file
diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 2535beec2..4cf701e69 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -37,7 +37,7 @@ patches:
             sysctl --system
             export DEBIAN_FRONTEND=noninteractive
             apt-get update -y
-            apt-get remove -y docker docker-engine docker.io containerd runc
+            apt-get remove -y docker docker-engine containerd runc
             apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
             mkdir -p /etc/apt/keyrings
             curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -47,14 +47,14 @@ patches:
             apt-get update -y
             TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
             RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-            apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+            apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
             cat  <<EOF > /etc/crictl.yaml
             runtime-endpoint: unix:///run/containerd/containerd.sock
             image-endpoint: unix:///run/containerd/containerd.sock
             EOF
             containerd config default > /etc/containerd/config.toml
             sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-            sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+            sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
             ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
@@ -113,7 +113,7 @@ patches:
               sysctl --system
               export DEBIAN_FRONTEND=noninteractive
               apt-get update -y
-              apt-get remove -y docker docker-engine docker.io containerd runc
+              apt-get remove -y docker docker-engine containerd runc
               apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
               mkdir -p /etc/apt/keyrings
               curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -123,12 +123,12 @@ patches:
               apt-get update -y
               TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
               RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-              apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+              apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
               cat  <<EOF > /etc/crictl.yaml
               runtime-endpoint: unix:///run/containerd/containerd.sock
               image-endpoint: unix:///run/containerd/containerd.sock
               EOF
               containerd config default > /etc/containerd/config.toml
               sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-              sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+              sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
               systemctl restart containerd
\ No newline at end of file

From 10fdaa038996697f14d498185384212c4a4a8cca Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Fri, 9 Sep 2022 15:44:04 -0500
Subject: [PATCH 10/16] ran make generate

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template-crs-cni.yaml          | 12 ++++++------
 templates/cluster-template-kube-vip-crs-cni.yaml | 12 ++++++------
 templates/cluster-template-kube-vip.yaml         | 12 ++++++------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/templates/cluster-template-crs-cni.yaml b/templates/cluster-template-crs-cni.yaml
index c6f44eb6e..83e288e8c 100644
--- a/templates/cluster-template-crs-cni.yaml
+++ b/templates/cluster-template-crs-cni.yaml
@@ -48,7 +48,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -58,14 +58,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -187,7 +187,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
       if [[ "$major_vers" -ge 20 ]]; then
@@ -201,14 +201,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       containerd config default > /etc/containerd/config.toml
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
   machineTemplate:
diff --git a/templates/cluster-template-kube-vip-crs-cni.yaml b/templates/cluster-template-kube-vip-crs-cni.yaml
index 4d3cd9a78..2995a60fa 100644
--- a/templates/cluster-template-kube-vip-crs-cni.yaml
+++ b/templates/cluster-template-kube-vip-crs-cni.yaml
@@ -48,7 +48,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -58,14 +58,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -180,7 +180,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       mkdir -p /etc/apt/keyrings
       curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -190,14 +190,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       containerd config default > /etc/containerd/config.toml
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
       curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
diff --git a/templates/cluster-template-kube-vip.yaml b/templates/cluster-template-kube-vip.yaml
index 244c30a08..7cf3d9f4b 100644
--- a/templates/cluster-template-kube-vip.yaml
+++ b/templates/cluster-template-kube-vip.yaml
@@ -29,7 +29,7 @@ spec:
         sysctl --system
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -y
-        apt-get remove -y docker docker-engine docker.io containerd runc
+        apt-get remove -y docker docker-engine containerd runc
         apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
         mkdir -p /etc/apt/keyrings
         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -39,14 +39,14 @@ spec:
         apt-get update -y
         TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
         RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-        apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
         cat  <<EOF > /etc/crictl.yaml
         runtime-endpoint: unix:///run/containerd/containerd.sock
         image-endpoint: unix:///run/containerd/containerd.sock
         EOF
         containerd config default > /etc/containerd/config.toml
         sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-        sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
         systemctl restart containerd
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -159,7 +159,7 @@ spec:
       sysctl --system
       export DEBIAN_FRONTEND=noninteractive
       apt-get update -y
-      apt-get remove -y docker docker-engine docker.io containerd runc
+      apt-get remove -y docker docker-engine containerd runc
       apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
       mkdir -p /etc/apt/keyrings
       curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
@@ -169,14 +169,14 @@ spec:
       apt-get update -y
       TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
       RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
-      apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
       cat  <<EOF > /etc/crictl.yaml
       runtime-endpoint: unix:///run/containerd/containerd.sock
       image-endpoint: unix:///run/containerd/containerd.sock
       EOF
       containerd config default > /etc/containerd/config.toml
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      sed -i "s,sandbox_image.*$,sandbox_image = $(kubeadm config images list | grep pause | sort -r | head -n1)," /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
       ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
       curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata

From edf12baa8c578ac70c6708b23efdbfbab92cbccb Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:15:52 -0500
Subject: [PATCH 11/16] Update defaults list

Removed things that were required and had no defaults
Added a variable for CPEM verison

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 README.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 8df27d63d..5754c68a6 100644
--- a/README.md
+++ b/README.md
@@ -34,13 +34,11 @@ You should then follow the [Cluster API Quick Start Guide](https://cluster-api.s
 
 If you do not change the generated `yaml` files, it will use defaults. You can look in the [templates/cluster-template.yaml](./templates/cluster-template.yaml) file for details.
 
-* `CLUSTER_NAME`                 (defaults to `my-cluster`)
-* `CONTROL_PLANE_MACHINE_COUNT`  (defaults to `1`)
+* `CPEM_VERSION`                 (defaults to `v3.5.0`)
 * `KUBE_VIP_VERSION`             (defaults to `v0.5.0`)
 * `NODE_OS`                      (defaults to `ubuntu_18_04`)
 * `POD_CIDR`                     (defaults to `192.168.0.0/16`)
 * `SERVICE_CIDR`                 (defaults to `172.26.0.0/16`)
-* `WORKER_MACHINE_COUNT`         (defaults to `0`)
   
 ### Reserved Hardware
 

From 5f8f05d50b45501855d3a7e95e50427a9a0147fd Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:22:01 -0500
Subject: [PATCH 12/16] Fix environment setup

Move the commands to run on all control plane nodes
Remove unnecessary KUBECONFIG export to save a line

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template.yaml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
index 81984c0ac..85ae09384 100644
--- a/templates/cluster-template.yaml
+++ b/templates/cluster-template.yaml
@@ -82,13 +82,12 @@ spec:
           netmask 255.255.255.255
         EOF
         systemctl restart networking
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
         if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-          export KUBECONFIG=/etc/kubernetes/admin.conf
-          mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-          echo "source <(kubectl completion bash)" >> /root/.bashrc
-          echo "alias k=kubectl" >> /root/.bashrc
-          echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+          export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
           export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
           kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
           kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

From 9a8229763e558a950ac4988514cb10c75a4d8014 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Mon, 12 Sep 2022 17:23:29 -0500
Subject: [PATCH 13/16] Fix kube-vip setup

Move the kube-vip manifest setup to after kubeadm is run,
this avoids a situation where kube-vip fights with previous
control plane nodes before they've joined the cluster.
Move the crictl/kubectl commands to run on all control plane nodes
Remove unnecessary KUBECONFIG export to save a line

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 .../experimental-kube-vip/kustomization.yaml      | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 4cf701e69..51dafac9b 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -57,6 +57,8 @@ patches:
             sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
             ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+        postKubeadmCommands:
+          - |
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
             for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
               ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
@@ -73,15 +75,12 @@ patches:
             --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
             --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
             rm /run/metadata.json
-        postKubeadmCommands:
-          - |
+            mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+            echo "source <(kubectl completion bash)" >> /root/.bashrc
+            echo "alias k=kubectl" >> /root/.bashrc
+            echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-              export KUBECONFIG=/etc/kubernetes/admin.conf
-              mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-              echo "source <(kubectl completion bash)" >> /root/.bashrc
-              echo "alias k=kubectl" >> /root/.bashrc
-              echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+              export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
               export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
               kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
               kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})

From 9876764d38ecff538d8b39820f729c3cc171c6b7 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Tue, 13 Sep 2022 14:11:57 -0500
Subject: [PATCH 14/16] Only setup kube-vip before kubeadm on first node

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 .../experimental-kube-vip/kustomization.yaml  | 53 +++++++++++++------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/templates/experimental-kube-vip/kustomization.yaml b/templates/experimental-kube-vip/kustomization.yaml
index 51dafac9b..d0bd53955 100644
--- a/templates/experimental-kube-vip/kustomization.yaml
+++ b/templates/experimental-kube-vip/kustomization.yaml
@@ -56,34 +56,53 @@ patches:
             sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
             sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
             systemctl restart containerd
-            ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+            if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+              ip addr add {{ .controlPlaneEndpoint }} dev lo
+              curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+              for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+                ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+              done
+              KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+              ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+              ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+              --interface "lo" \
+              --vip "{{ .controlPlaneEndpoint }}" \
+              --controlplane \
+              --bgp \
+              --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+              --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+              --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+              --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+              rm /run/metadata.json
+            fi
         postKubeadmCommands:
           - |
             curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
             for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
               ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
             done
-            KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-            ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-            ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-            --interface "lo" \
-            --vip "{{ .controlPlaneEndpoint }}" \
-            --controlplane \
-            --bgp \
-            --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-            --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-            --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-            --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-            rm /run/metadata.json
-            mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
-            echo "source <(kubectl completion bash)" >> /root/.bashrc
-            echo "alias k=kubectl" >> /root/.bashrc
-            echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
               export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
               export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
               kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
               kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+            else
+              KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+              ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+              ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+              --interface "lo" \
+              --vip "{{ .controlPlaneEndpoint }}" \
+              --controlplane \
+              --bgp \
+              --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+              --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+              --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+              --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+              rm /run/metadata.json
+              mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+              echo "source <(kubectl completion bash)" >> /root/.bashrc
+              echo "alias k=kubectl" >> /root/.bashrc
+              echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
             fi
 - patch: |
     kind: KubeadmConfigTemplate

From bf28d9319177fa542da104654eaaf5df8a5bba52 Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Tue, 13 Sep 2022 14:12:10 -0500
Subject: [PATCH 15/16] Run make generate

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 templates/cluster-template-crs-cni.yaml       | 11 ++--
 .../cluster-template-kube-vip-crs-cni.yaml    | 64 ++++++++++++-------
 templates/cluster-template-kube-vip.yaml      | 64 ++++++++++++-------
 3 files changed, 87 insertions(+), 52 deletions(-)

diff --git a/templates/cluster-template-crs-cni.yaml b/templates/cluster-template-crs-cni.yaml
index 83e288e8c..9500abcb3 100644
--- a/templates/cluster-template-crs-cni.yaml
+++ b/templates/cluster-template-crs-cni.yaml
@@ -157,13 +157,12 @@ spec:
         netmask 255.255.255.255
       EOF
       systemctl restart networking
+      mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+      echo "source <(kubectl completion bash)" >> /root/.bashrc
+      echo "alias k=kubectl" >> /root/.bashrc
+      echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
diff --git a/templates/cluster-template-kube-vip-crs-cni.yaml b/templates/cluster-template-kube-vip-crs-cni.yaml
index 2995a60fa..541bcd100 100644
--- a/templates/cluster-template-kube-vip-crs-cni.yaml
+++ b/templates/cluster-template-kube-vip-crs-cni.yaml
@@ -150,16 +150,32 @@ spec:
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
     postKubeadmCommands:
     - |-
+      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+      done
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      else
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       fi
     preKubeadmCommands:
     - |
@@ -199,23 +215,25 @@ spec:
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
       sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
-      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
-      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
-      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
-        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
-      done
-      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-      --interface "lo" \
-      --vip "{{ .controlPlaneEndpoint }}" \
-      --controlplane \
-      --bgp \
-      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-      rm /run/metadata.json
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        ip addr add {{ .controlPlaneEndpoint }} dev lo
+        curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+        for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+          ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+        done
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+      fi
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
diff --git a/templates/cluster-template-kube-vip.yaml b/templates/cluster-template-kube-vip.yaml
index 7cf3d9f4b..832f45340 100644
--- a/templates/cluster-template-kube-vip.yaml
+++ b/templates/cluster-template-kube-vip.yaml
@@ -129,16 +129,32 @@ spec:
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
     postKubeadmCommands:
     - |-
+      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+      done
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        mkdir -p /root/.kube && cp -f $${KUBECONFIG} /root/.kube/config
-        echo "source <(kubectl completion bash)" >> /root/.bashrc
-        echo "alias k=kubectl" >> /root/.bashrc
-        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
-        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.5.0}/deployment.yaml
         export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}'''
         kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      else
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+        mkdir -p /root/.kube && cp -f /etc/kubernetes/admin.conf /root/.kube/config
+        echo "source <(kubectl completion bash)" >> /root/.bashrc
+        echo "alias k=kubectl" >> /root/.bashrc
+        echo "complete -o default -F __start_kubectl k" >> /root/.bashrc
       fi
     preKubeadmCommands:
     - |
@@ -178,23 +194,25 @@ spec:
       sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
       sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
       systemctl restart containerd
-      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
-      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
-      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
-        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
-      done
-      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
-      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
-      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
-      --interface "lo" \
-      --vip "{{ .controlPlaneEndpoint }}" \
-      --controlplane \
-      --bgp \
-      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
-      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
-      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
-      rm /run/metadata.json
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        ip addr add {{ .controlPlaneEndpoint }} dev lo
+        curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata
+        for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do
+          ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway')
+        done
+        KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}"
+        ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION}
+        ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \
+        --interface "lo" \
+        --vip "{{ .controlPlaneEndpoint }}" \
+        --controlplane \
+        --bgp \
+        --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \
+        --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \
+        --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \
+        --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml
+        rm /run/metadata.json
+      fi
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

From 5f522c3fbe65b9ea64c4902317d43b0279c01abd Mon Sep 17 00:00:00 2001
From: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
Date: Fri, 16 Sep 2022 14:18:18 -0500
Subject: [PATCH 16/16] Add tools bin to path for e2e testing's sake

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index ab399bbbd..c7161e627 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,7 @@ CURL_RETRIES=3
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 TOOLS_DIR := hack/tools
 TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
+PATH := $(TOOLS_BIN_DIR):$(PATH)
 BIN_DIR := $(abspath $(ROOT_DIR)/bin)
 GO_INSTALL = ./scripts/go_install.sh