feat: add new section to makefile to create cluster-template-developm…

…ent.yaml Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
kubernetes-sigs · Jun 26, 2024 · 1f44633 · 1f44633
1 parent 8a52232
commit 1f44633
Show file tree

Hide file tree

Showing 3 changed files with 249 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -283,6 +283,7 @@ generate: ## Generate code
 
 .PHONY: generate-templates
 generate-templates: $(KUSTOMIZE) ## Generate cluster templates
+	$(KUSTOMIZE) build templates/development --load-restrictor LoadRestrictionsNone > templates/cluster-template-development.yaml
 	$(KUSTOMIZE) build templates/experimental-emlb --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb.yaml
 	$(KUSTOMIZE) build templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb-crs-cni.yaml
 	$(KUSTOMIZE) build templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip-crs-cni.yaml

diff --git a/templates/cluster-template-development.yaml b/templates/cluster-template-development.yaml
@@ -0,0 +1,243 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+            cloud-provider: external
+            provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+      preKubeadmCommands:
+      - |
+        sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+        swapoff -a
+        mount -a
+        cat <<EOF > /etc/modules-load.d/containerd.conf
+        overlay
+        br_netfilter
+        EOF
+        modprobe overlay
+        modprobe br_netfilter
+        cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
+        net.bridge.bridge-nf-call-iptables  = 1
+        net.ipv4.ip_forward                 = 1
+        net.bridge.bridge-nf-call-ip6tables = 1
+        EOF
+        sysctl --system
+        export DEBIAN_FRONTEND=noninteractive
+        apt-get update -y
+        apt-get remove -y docker docker-engine containerd runc
+        apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+        install -m 0755 -d /etc/apt/keyrings
+        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+        MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
+        curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+        chmod a+r /etc/apt/keyrings/docker.gpg
+        chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+        echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
+        echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
+        apt-get update -y
+        TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+        RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
+        apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+        cat  <<EOF > /etc/crictl.yaml
+        runtime-endpoint: unix:///run/containerd/containerd.sock
+        image-endpoint: unix:///run/containerd/containerd.sock
+        EOF
+        containerd config default > /etc/containerd/config.toml
+        sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+        sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
+        systemctl restart containerd
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - ${POD_CIDR:=192.168.0.0/16}
+    services:
+      cidrBlocks:
+      - ${SERVICE_CIDR:=172.26.0.0/16}
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KubeadmControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: PacketCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+    pool: worker-a
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+      pool: worker-a
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+        pool: worker-a
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-worker-a
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: PacketMachineTemplate
+        name: ${CLUSTER_NAME}-worker-a
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      apiServer:
+        extraArgs:
+          cloud-provider: external
+      controllerManager:
+        extraArgs:
+          cloud-provider: external
+    initConfiguration:
+      nodeRegistration:
+        kubeletExtraArgs:
+          cloud-provider: external
+          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+    joinConfiguration:
+      nodeRegistration:
+        ignorePreflightErrors:
+        - DirAvailable--etc-kubernetes-manifests
+        kubeletExtraArgs:
+          cloud-provider: external
+          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
+    postKubeadmCommands:
+    - |
+      cat <<EOF >> /etc/network/interfaces
+      auto lo:0
+      iface lo:0 inet static
+        address {{ .controlPlaneEndpoint }}
+        netmask 255.255.255.255
+      EOF
+      systemctl restart networking
+      mkdir -p $HOME/.kube
+      cp /etc/kubernetes/admin.conf $HOME/.kube/config
+      echo "source <(kubectl completion bash)" >> $HOME/.bashrc
+      echo "alias k=kubectl" >> $HOME/.bashrc
+      echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc
+      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
+        export KUBECONFIG=/etc/kubernetes/admin.conf
+        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.7.0}/deployment.yaml
+        export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
+        kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
+        kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
+      fi
+    preKubeadmCommands:
+    - |
+      sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+      swapoff -a
+      mount -a
+      cat <<EOF > /etc/modules-load.d/containerd.conf
+      overlay
+      br_netfilter
+      EOF
+      modprobe overlay
+      modprobe br_netfilter
+      cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.ipv4.ip_forward                 = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+      EOF
+      sysctl --system
+      export DEBIAN_FRONTEND=noninteractive
+      apt-get update -y
+      apt-get remove -y docker docker-engine containerd runc
+      apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+      major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
+      if [ "$major_vers" -ge 20 ]; then
+        apt-get install -y kubetail
+      fi
+      install -m 0755 -d /etc/apt/keyrings
+      curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+      MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
+      curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+      chmod a+r /etc/apt/keyrings/docker.gpg
+      chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+      echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
+      echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
+      apt-get update -y
+      TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+      RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
+      apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      containerd config default > /etc/containerd/config.toml
+      cat  <<EOF > /etc/crictl.yaml
+      runtime-endpoint: unix:///run/containerd/containerd.sock
+      image-endpoint: unix:///run/containerd/containerd.sock
+      EOF
+      sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+      sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
+      systemctl restart containerd
+      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: PacketMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  metro: ${METRO}
+  projectID: ${PROJECT_ID}
+  vipManager: CPEM
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      billingCycle: hourly
+      machineType: ${CONTROLPLANE_NODE_TYPE}
+      os: ${NODE_OS:=ubuntu_20_04}
+      sshKeys:
+      - ${SSH_KEY}
+      tags: []
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-worker-a
+spec:
+  template:
+    spec:
+      billingCycle: hourly
+      machineType: ${WORKER_NODE_TYPE}
+      os: ${NODE_OS:=ubuntu_20_04}
+      sshKeys:
+      - ${SSH_KEY}
+      tags: []
diff --git a/templates/development/kustomization.yaml b/templates/development/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../cluster-template.yaml