✨ Add support for kube-vip

Makefile

@@ -275,6 +275,7 @@ generate: ## Generate code
 
 .PHONY: generate-templates
 generate-templates: $(KUSTOMIZE) ## Generate cluster templates
+	$(KUSTOMIZE) build templates/experimental-kube-vip --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip.yaml
 	$(KUSTOMIZE) build templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-crs-cni.yaml
 	$(KUSTOMIZE) build templates/addons/calico > templates/addons/calico.yaml
 

config/default/config.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: manager-config
+  namespace: system
+data:
+  EIP_MANAGEMENT: "${EIP_MANAGEMENT:=CPEM}"

config/default/kustomization.yaml

@@ -12,6 +12,7 @@ commonLabels:
 
 resources:
 - namespace.yaml
+- config.yaml
 - credentials.yaml
 
 bases:
@@ -26,7 +27,7 @@ patchesStrategicMerge:
 - manager_pull_policy.yaml
 - manager_webhook_patch.yaml
 - webhookcainjection_patch.yaml
-- manager_credentials_patch.yaml
+- manager_credentials_config_patch.yaml
 
 vars:
   - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR

config/default/kustomizeconfig.yaml

@@ -2,3 +2,5 @@
 varReference:
 - kind: Deployment
   path: spec/template/spec/volumes/secret/secretName
+- kind: Deployment
+  path: spec/template/spec/volumes/configMap/configMapName

config/default/manager_credentials_patch.yaml → config/default/manager_credentials_config_patch.yaml

@@ -11,3 +11,5 @@ spec:
         envFrom:
         - secretRef:
             name: manager-api-credentials
+        - configMapRef:
+            name: manager-config

controllers/packetcluster_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"errors"
+	"os"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -137,6 +138,13 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
 		}
 	}
 
+	if os.Getenv("EIP_MANAGEMENT") == "KUBE_VIP" {
+		if err := r.PacketClient.EnableProjectBGP(packetCluster.Spec.ProjectID); err != nil {
+			log.Error(err, "error enabling bgp for project")
+			return ctrl.Result{}, err
+		}
+	}
+
 	clusterScope.PacketCluster.Status.Ready = true
 	conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
 

controllers/packetmachine_controller.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -222,7 +223,7 @@ func (r *PacketMachineReconciler) PacketClusterToPacketMachines(ctx context.Cont
 	}
 }
 
-func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) { //nolint:gocyclo
+func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx, "machine", machineScope.Machine.Name, "cluster", machineScope.Cluster.Name)
 	log.Info("Reconciling PacketMachine")
 
@@ -316,21 +317,21 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 			ExtraTags:    packet.DefaultCreateTags(machineScope.Namespace(), machineScope.Machine.Name, machineScope.Cluster.Name),
 		}
 
-		// TODO: see if this can be removed with kube-vip in place
-		// when the node is a control plan we should check if the elastic ip
-		// for this cluster is not assigned. If it is free we can prepare the
-		// current node to use it.
+		// when the node is a control plan we need the elastic IP
+		// to template out the kube-vip deployment
 		if machineScope.IsControlPlane() {
 			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
 				machineScope.Cluster.Namespace,
 				machineScope.Cluster.Name,
 				machineScope.PacketCluster.Spec.ProjectID)
-			if len(controlPlaneEndpoint.Assignments) == 0 {
-				a := corev1.NodeAddress{
-					Type:    corev1.NodeExternalIP,
-					Address: controlPlaneEndpoint.Address,
+			if os.Getenv("EIP_MANAGEMENT") == "CPEM" {
+				if len(controlPlaneEndpoint.Assignments) == 0 {
+					a := corev1.NodeAddress{
+						Type:    corev1.NodeExternalIP,
+						Address: controlPlaneEndpoint.Address,
+					}
+					addrs = append(addrs, a)
 				}
-				addrs = append(addrs, a)
 			}
 			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.Address
 		}
@@ -362,6 +363,13 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	machineScope.SetProviderID(dev.ID)
 	machineScope.SetInstanceStatus(infrav1.PacketResourceStatus(dev.State))
 
+	if os.Getenv("EIP_MANAGEMENT") == "KUBE_VIP" {
+		if err := r.PacketClient.EnsureNodeBGPEnabled(dev.ID); err != nil {
+			// Do not treat an error enabling bgp on machine as fatal
+			return ctrl.Result{RequeueAfter: time.Second * 20}, fmt.Errorf("failed to enable bpg on machine %s: %w", machineScope.Name(), err)
+		}
+	}
+
 	deviceAddr := r.PacketClient.GetDeviceAddresses(dev)
 	machineScope.SetAddresses(append(addrs, deviceAddr...))
 
@@ -376,22 +384,21 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	case infrav1.PacketResourceStatusRunning:
 		log.Info("Machine instance is active", "instance-id", machineScope.GetInstanceID())
 
-		// TODO: see if this can be removed with kube-vip in place
-		// This logic is here because an elastic ip can be assigned only an
-		// active node. It needs to be a control plane and the IP should not be
-		// assigned to anything at this point.
-		controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
-			machineScope.Cluster.Namespace,
-			machineScope.Cluster.Name,
-			machineScope.PacketCluster.Spec.ProjectID)
-		if len(controlPlaneEndpoint.Assignments) == 0 && machineScope.IsControlPlane() {
-			if _, _, err := r.PacketClient.DeviceIPs.Assign(dev.ID, &packngo.AddressStruct{
-				Address: controlPlaneEndpoint.Address,
-			}); err != nil {
-				log.Error(err, "err assigining elastic ip to control plane. retrying...")
-				return ctrl.Result{RequeueAfter: time.Second * 20}, nil
+		if os.Getenv("EIP_MANAGEMENT") == "CPEM" {
+			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
+				machineScope.Cluster.Namespace,
+				machineScope.Cluster.Name,
+				machineScope.PacketCluster.Spec.ProjectID)
+			if len(controlPlaneEndpoint.Assignments) == 0 && machineScope.IsControlPlane() {
+				if _, _, err := r.PacketClient.DeviceIPs.Assign(dev.ID, &packngo.AddressStruct{
+					Address: controlPlaneEndpoint.Address,
+				}); err != nil {
+					log.Error(err, "err assigining elastic ip to control plane. retrying...")
+					return ctrl.Result{RequeueAfter: time.Second * 20}, nil
+				}
 			}
 		}
+
 		machineScope.SetReady()
 		conditions.MarkTrue(machineScope.PacketMachine, infrav1.DeviceReadyCondition)
 

pkg/cloud/packet/client.go

@@ -23,6 +23,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"text/template"
 
@@ -38,6 +39,9 @@ const (
 	apiTokenVarName = "PACKET_API_KEY" //nolint:gosec
 	clientName      = "CAPP-v1beta1"
 	ipxeOS          = "custom_ipxe"
+	envVarLocalASN  = "METAL_LOCAL_ASN"
+	envVarBGPPass   = "METAL_BGP_PASS" //nolint:gosec
+	DefaultLocalASN = 65000
 )
 
 var (
@@ -230,6 +234,68 @@ func (p *Client) CreateIP(namespace, clusterName, projectID, facility string) (n
 	return ip, nil
 }
 
+// enableBGP enable bgp on the project
+func (p *Client) EnableProjectBGP(projectID string) error {
+	// first check if it is enabled before trying to create it
+	bgpConfig, _, err := p.BGPConfig.Get(projectID, &packngo.GetOptions{})
+	// if we already have a config, just return
+	// we need some extra handling logic because the API always returns 200, even if
+	// not BGP config is in place.
+	// We treat it as valid config already exists only if ALL of the above is true:
+	// - no error
+	// - bgpConfig struct exists
+	// - bgpConfig struct has non-blank ID
+	// - bgpConfig struct does not have Status=="disabled"
+	if err == nil && bgpConfig != nil && bgpConfig.ID != "" && strings.ToLower(bgpConfig.Status) != "disabled" {
+		return nil
+	}
+
+	// get the local ASN
+	localASN := os.Getenv(envVarLocalASN)
+	var outLocalASN int
+	switch {
+	case localASN != "":
+		localASNNo, err := strconv.Atoi(localASN)
+		if err != nil {
+			return fmt.Errorf("env var %s must be a number, was %s: %w", envVarLocalASN, localASN, err)
+		}
+		outLocalASN = localASNNo
+	default:
+		outLocalASN = DefaultLocalASN
+	}
+
+	var outBGPPass string
+	bgpPass := os.Getenv(envVarBGPPass)
+	if bgpPass != "" {
+		outBGPPass = bgpPass
+	}
+
+	// we did not have a valid one, so create it
+	req := packngo.CreateBGPConfigRequest{
+		Asn:            outLocalASN,
+		Md5:            outBGPPass,
+		DeploymentType: "local",
+		UseCase:        "kubernetes-load-balancer",
+	}
+	_, err = p.BGPConfig.Create(projectID, req)
+	return err
+}
+
+// ensureNodeBGPEnabled check if the node has bgp enabled, and set it if it does not
+func (p *Client) EnsureNodeBGPEnabled(id string) error {
+	// fortunately, this is idempotent, so just create
+	req := packngo.CreateBGPSessionRequest{
+		AddressFamily: "ipv4",
+	}
+	_, response, err := p.BGPSessions.Create(id, req)
+	// if we already had one, then we can ignore the error
+	// this really should be a 409, but 422 is what is returned
+	if response.StatusCode == 422 && strings.Contains(fmt.Sprintf("%s", err), "already has session") {
+		err = nil
+	}
+	return err
+}
+
 func (p *Client) GetIPByClusterIdentifier(namespace, name, projectID string) (packngo.IPAddressReservation, error) {
 	var err error
 	var reservedIP packngo.IPAddressReservation

templates/cluster-template-crs-cni.yaml

@@ -30,34 +30,30 @@ spec:
             cloud-provider: external
             provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
       preKubeadmCommands:
-      - sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
-      - swapoff -a
-      - mount -a
       - |
+        sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+        swapoff -a
+        mount -a
         cat <<EOF > /etc/modules-load.d/containerd.conf
         overlay
         br_netfilter
         EOF
-      - modprobe overlay
-      - modprobe br_netfilter
-      - |
+        modprobe overlay
+        modprobe br_netfilter
         cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
         net.bridge.bridge-nf-call-iptables  = 1
         net.ipv4.ip_forward                 = 1
         net.bridge.bridge-nf-call-ip6tables = 1
         EOF
-      - sysctl --system
-      - apt-get -y update
-      - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
-      - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-      - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
-      - apt-get update -y
-      - TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\./g' | sed 's/^v//')
-      - RESOLVED_KUBERNETES_VERSION=$(apt-cache policy kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$1~ VERSION { print $1 }' | head -n1)
-      - apt-get install -y ca-certificates socat jq ebtables apt-transport-https cloud-utils prips containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
-      - systemctl daemon-reload
-      - systemctl enable containerd
-      - systemctl start containerd
+        sysctl --system
+        apt-get -y update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+        curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+        echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+        apt-get update -y
+        TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+        RESOLVED_KUBERNETES_VERSION=$(apt-cache policy kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$1~ VERSION { print $1 }' | head -n1)
+        DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
@@ -134,7 +130,8 @@ spec:
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
     joinConfiguration:
       nodeRegistration:
-        ignorePreflightErrors: []
+        ignorePreflightErrors:
+        - DirAvailable--etc-kubernetes-manifests
         kubeletExtraArgs:
           cloud-provider: external
           provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
@@ -146,8 +143,6 @@ spec:
         address {{ .controlPlaneEndpoint }}
         netmask 255.255.255.255
       EOF
-    - systemctl restart networking
-    - |
       if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
         export KUBECONFIG=/etc/kubernetes/admin.conf
         export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.4.0/deployment.yaml
@@ -156,35 +151,31 @@ spec:
         kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
       fi
     preKubeadmCommands:
-    - sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
-    - swapoff -a
-    - mount -a
     - |
+      sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+      swapoff -a
+      mount -a
       cat <<EOF > /etc/modules-load.d/containerd.conf
       overlay
       br_netfilter
       EOF
-    - modprobe overlay
-    - modprobe br_netfilter
-    - |
+      modprobe overlay
+      modprobe br_netfilter
       cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
       net.bridge.bridge-nf-call-iptables  = 1
       net.ipv4.ip_forward                 = 1
       net.bridge.bridge-nf-call-ip6tables = 1
       EOF
-    - sysctl --system
-    - apt-get -y update
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
-    - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-    - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
-    - apt-get update -y
-    - TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\./g' | sed 's/^v//')
-    - RESOLVED_KUBERNETES_VERSION=$(apt-cache policy kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$1~ VERSION { print $1 }' | head -n1)
-    - apt-get install -y ca-certificates socat jq ebtables apt-transport-https cloud-utils prips containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
-    - systemctl daemon-reload
-    - systemctl enable containerd
-    - systemctl start containerd
-    - ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
+      sysctl --system
+      apt-get -y update
+      DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
+      curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+      echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+      apt-get update -y
+      TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
+      RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
+      DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
+      ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1