feat: Add Equinix Metal Load Balancer support

- Add internal package for interacting with Equinix Metal Load Balancer API
- packetcluster_controller creates load balancer and listener port and stores their ids in packetCluster annotations.
- packetmachine_controller creates an origin pool and origin port for each machine and stores their IDs in the packetMachine annotations.
- CPEMLBConfig and EMLBID added to the packet cloud client package to be able to provide a config for the CPEM loadBalancer setting in the emlb templates.
- Memory request for the Cluster API Provider Packet controller increased to 300Mi to avoid OOMing while debugging.
- EMLB added as a valid VIPManager enum type.

Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>

.golangci.yml

@@ -164,7 +164,6 @@ linters-settings:
         alias: infraexpv1
   nolintlint:
     allow-unused: false
-    allow-leading-space: false
     require-specific: true
   revive:
     rules:
@@ -314,3 +313,7 @@ issues:
         - gocritic
       text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
       path: _test\.go
+    - linters:
+        - bodyclose
+      path: .*(internal)/emlb/emlb.go
+      text: "response body must be closed"

api/v1beta1/packetcluster_types.go

@@ -26,7 +26,7 @@ const (
 	NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
 )
 
-// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip.
+// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer.
 type VIPManagerType string
 
 // PacketClusterSpec defines the desired state of PacketCluster.
@@ -46,9 +46,9 @@ type PacketClusterSpec struct {
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
 
-	// VIPManager represents whether this cluster uses CPEM or kube-vip to
+	// VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
 	// manage its vip for the api server IP
-	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
+	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB
 	// +kubebuilder:default:=CPEM
 	VIPManager VIPManagerType `json:"vipManager"`
 }

clusterctl-settings.json

@@ -2,7 +2,7 @@
     "name": "infrastructure-packet",
     "config": {
       "componentsFile": "infrastructure-components.yaml",
-      "nextVersion": "v0.6.99"
+      "nextVersion": "v0.8.99"
     }
   }
 

config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml

@@ -75,10 +75,12 @@ spec:
               vipManager:
                 default: CPEM
                 description: VIPManager represents whether this cluster uses CPEM
-                  or kube-vip to manage its vip for the api server IP
+                  or kube-vip or Equinix Metal Load Balancer to manage its vip for
+                  the api server IP
                 enum:
                 - CPEM
                 - KUBE_VIP
+                - EMLB
                 type: string
             required:
             - projectID

config/manager/manager.yaml

@@ -42,10 +42,10 @@ spec:
               port: healthz
           resources:
             limits:
-              memory: 200Mi
+              memory: 300Mi
             requests:
               cpu: 100m
-              memory: 200Mi
+              memory: 300Mi
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

controllers/packetcluster_controller.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 )
@@ -114,49 +115,63 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
 
 	packetCluster := clusterScope.PacketCluster
 
-	ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
 	switch {
-	case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
-		// Parse metro and facility from the cluster spec
-		var metro, facility string
-
-		facility = packetCluster.Spec.Facility
-		metro = packetCluster.Spec.Metro
-
-		// If both specified, metro takes precedence over facility
-		if metro != "" {
-			facility = ""
+	case packetCluster.Spec.VIPManager == "EMLB":
+		if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() {
+			// Create new EMLB object
+			lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro)
+
+			if err := lb.ReconcileLoadBalancer(ctx, clusterScope); err != nil {
+				log.Error(err, "Error Reconciling EMLB")
+				return err
+			}
 		}
-
-		// There is not an ElasticIP with the right tags, at this point we can create one
-		ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
-		if err != nil {
-			log.Error(err, "error reserving an ip")
+	case packetCluster.Spec.VIPManager == "KUBE_VIP":
+		log.Info("KUBE_VIP VIPManager Detected")
+		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
+			log.Error(err, "error enabling bgp for project")
 			return err
 		}
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ip.To4().String(),
-			Port: 6443,
-		}
-	case err != nil:
-		log.Error(err, "error getting cluster IP")
-		return err
-	default:
-		// If there is an ElasticIP with the right tag just use it again
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ipReserv.GetAddress(),
-			Port: 6443,
-		}
 	}
 
-	if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
-		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
-			log.Error(err, "error enabling bgp for project")
+	if packetCluster.Spec.VIPManager != "EMLB" {
+		ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
+		switch {
+		case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
+			// Parse metro and facility from the cluster spec
+			var metro, facility string
+
+			facility = packetCluster.Spec.Facility
+			metro = packetCluster.Spec.Metro
+
+			// If both specified, metro takes precedence over facility
+			if metro != "" {
+				facility = ""
+			}
+
+			// There is not an ElasticIP with the right tags, at this point we can create one
+			ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
+			if err != nil {
+				log.Error(err, "error reserving an ip")
+				return err
+			}
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ip.To4().String(),
+				Port: 6443,
+			}
+		case err != nil:
+			log.Error(err, "error getting cluster IP")
 			return err
+		default:
+			// If there is an ElasticIP with the right tag just use it again
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ipReserv.GetAddress(),
+				Port: 6443,
+			}
 		}
 	}
 
-	clusterScope.PacketCluster.Status.Ready = true
+	packetCluster.Status.Ready = true
 	conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
 
 	return nil

controllers/packetmachine_controller.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 	clog "sigs.k8s.io/cluster-api/util/log"
@@ -348,23 +349,33 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 		// when a node is a control plane node we need the elastic IP
 		// to template out the kube-vip deployment
 		if machineScope.IsControlPlane() {
-			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
-				ctx,
-				machineScope.Cluster.Namespace,
-				machineScope.Cluster.Name,
-				machineScope.PacketCluster.Spec.ProjectID)
-			if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+			var controlPlaneEndpointAddress string
+			var cpemLBConfig string
+			var emlbID string
+			switch {
+			case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
+				controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
+					ctx,
+					machineScope.Cluster.Namespace,
+					machineScope.Cluster.Name,
+					machineScope.PacketCluster.Spec.ProjectID)
 				if len(controlPlaneEndpoint.Assignments) == 0 {
 					a := corev1.NodeAddress{
 						Type:    corev1.NodeExternalIP,
 						Address: controlPlaneEndpoint.GetAddress(),
 					}
 					addrs = append(addrs, a)
 				}
+				controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress()
+			case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
+				controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host
+				cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro
+				emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"]
 			}
-			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.GetAddress()
+			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpointAddress
+			createDeviceReq.CPEMLBConfig = cpemLBConfig
+			createDeviceReq.EMLBID = emlbID
 		}
-
 		dev, err = r.PacketClient.NewDevice(ctx, createDeviceReq)
 
 		switch {
@@ -413,7 +424,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	case infrav1.PacketResourceStatusRunning:
 		log.Info("Machine instance is active", "instance-id", machineScope.ProviderID())
 
-		if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+		switch {
+		case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
 			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
 				ctx,
 				machineScope.Cluster.Namespace,
@@ -428,6 +440,15 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 					return ctrl.Result{RequeueAfter: time.Second * 20}, nil
 				}
 			}
+		case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
+			if machineScope.IsControlPlane() {
+				// Create new EMLB object
+				lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro)
+
+				if err := lb.ReconcileVIPOrigin(ctx, machineScope, deviceAddr); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
 		}
 
 		machineScope.SetReady()

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
+	golang.org/x/oauth2 v0.14.0
 	k8s.io/api v0.28.7
 	k8s.io/apimachinery v0.28.7
 	k8s.io/client-go v0.28.7
@@ -81,7 +82,6 @@ require (
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect