Merge pull request #740 from cprivitere/add-emlb-support

✨ Add Equinix Metal Load Balancer support

.golangci.yml

@@ -164,7 +164,6 @@ linters-settings:
         alias: infraexpv1
   nolintlint:
     allow-unused: false
-    allow-leading-space: false
     require-specific: true
   revive:
     rules:
@@ -314,3 +313,7 @@ issues:
         - gocritic
       text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
       path: _test\.go
+    - linters:
+        - bodyclose
+      path: .*(internal)/emlb/emlb.go
+      text: "response body must be closed"

api/v1beta1/packetcluster_types.go

@@ -26,7 +26,7 @@ const (
 	NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
 )
 
-// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip.
+// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer.
 type VIPManagerType string
 
 // PacketClusterSpec defines the desired state of PacketCluster.
@@ -46,9 +46,9 @@ type PacketClusterSpec struct {
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
 
-	// VIPManager represents whether this cluster uses CPEM or kube-vip to
+	// VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
 	// manage its vip for the api server IP
-	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
+	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB
 	// +kubebuilder:default:=CPEM
 	VIPManager VIPManagerType `json:"vipManager"`
 }

api/v1beta1/packetcluster_webhook.go

@@ -94,7 +94,7 @@ func (c *PacketCluster) ValidateUpdate(oldRaw runtime.Object) (admission.Warning
 	}
 
 	// Must have only one of Metro or Facility
-	if len(c.Spec.Facility) > 0 && len(c.Spec.Metro) > 0 {
+	if c.Spec.Facility != "" && c.Spec.Metro != "" {
 		allErrs = append(allErrs,
 			field.Invalid(field.NewPath("spec", "Facility"),
 				c.Spec.Facility, "Metro and Facility are mutually exclusive, Metro is recommended"),

api/v1beta1/packetmachine_webhook.go

@@ -53,7 +53,7 @@ func (m *PacketMachine) ValidateUpdate(old runtime.Object) (admission.Warnings,
 	var allErrs field.ErrorList
 
 	// Must have only one of Metro or Facility specified
-	if len(m.Spec.Facility) > 0 && len(m.Spec.Metro) > 0 {
+	if m.Spec.Facility != "" && m.Spec.Metro != "" {
 		allErrs = append(allErrs,
 			field.Invalid(field.NewPath("spec", "Facility"),
 				m.Spec.Facility, "Metro and Facility field are mutually exclusive"),

clusterctl-settings.json

@@ -2,7 +2,7 @@
     "name": "infrastructure-packet",
     "config": {
       "componentsFile": "infrastructure-components.yaml",
-      "nextVersion": "v0.6.99"
+      "nextVersion": "v0.8.99"
     }
   }
 

cmd/ci-clean/main.go

@@ -43,7 +43,7 @@ func main() {
 	rootCmd := &cobra.Command{ //nolint:exhaustivestruct
 		Use:   "ci-clean",
 		Short: "Clean up any stray resources in CI",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			metalAuthToken := os.Getenv(authTokenEnvVar)
 			if metalAuthToken == "" {
 				return fmt.Errorf("%s: %w", authTokenEnvVar, errMissingRequiredEnvVar)

config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml

@@ -80,11 +80,12 @@ spec:
               vipManager:
                 default: CPEM
                 description: |-
-                  VIPManager represents whether this cluster uses CPEM or kube-vip to
+                  VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
                   manage its vip for the api server IP
                 enum:
                 - CPEM
                 - KUBE_VIP
+                - EMLB
                 type: string
             required:
             - projectID

config/manager/manager.yaml

@@ -42,10 +42,10 @@ spec:
               port: healthz
           resources:
             limits:
-              memory: 200Mi
+              memory: 300Mi
             requests:
               cpu: 100m
-              memory: 200Mi
+              memory: 300Mi
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

controllers/packetcluster_controller.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 )
@@ -114,49 +115,63 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
 
 	packetCluster := clusterScope.PacketCluster
 
-	ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
 	switch {
-	case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
-		// Parse metro and facility from the cluster spec
-		var metro, facility string
-
-		facility = packetCluster.Spec.Facility
-		metro = packetCluster.Spec.Metro
-
-		// If both specified, metro takes precedence over facility
-		if metro != "" {
-			facility = ""
+	case packetCluster.Spec.VIPManager == emlb.EMLBVIPID:
+		if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() {
+			// Create new EMLB object
+			lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro)
+
+			if err := lb.ReconcileLoadBalancer(ctx, clusterScope); err != nil {
+				log.Error(err, "Error Reconciling EMLB")
+				return err
+			}
 		}
-
-		// There is not an ElasticIP with the right tags, at this point we can create one
-		ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
-		if err != nil {
-			log.Error(err, "error reserving an ip")
+	case packetCluster.Spec.VIPManager == "KUBE_VIP":
+		log.Info("KUBE_VIP VIPManager Detected")
+		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
+			log.Error(err, "error enabling bgp for project")
 			return err
 		}
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ip.To4().String(),
-			Port: 6443,
-		}
-	case err != nil:
-		log.Error(err, "error getting cluster IP")
-		return err
-	default:
-		// If there is an ElasticIP with the right tag just use it again
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ipReserv.GetAddress(),
-			Port: 6443,
-		}
 	}
 
-	if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
-		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
-			log.Error(err, "error enabling bgp for project")
+	if packetCluster.Spec.VIPManager != emlb.EMLBVIPID {
+		ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
+		switch {
+		case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
+			// Parse metro and facility from the cluster spec
+			var metro, facility string
+
+			facility = packetCluster.Spec.Facility
+			metro = packetCluster.Spec.Metro
+
+			// If both specified, metro takes precedence over facility
+			if metro != "" {
+				facility = ""
+			}
+
+			// There is not an ElasticIP with the right tags, at this point we can create one
+			ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
+			if err != nil {
+				log.Error(err, "error reserving an ip")
+				return err
+			}
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ip.To4().String(),
+				Port: 6443,
+			}
+		case err != nil:
+			log.Error(err, "error getting cluster IP")
 			return err
+		default:
+			// If there is an ElasticIP with the right tag just use it again
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ipReserv.GetAddress(),
+				Port: 6443,
+			}
 		}
 	}
 
-	clusterScope.PacketCluster.Status.Ready = true
+	packetCluster.Status.Ready = true
 	conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
 
 	return nil

controllers/packetmachine_controller.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 	clog "sigs.k8s.io/cluster-api/util/log"
@@ -348,23 +349,33 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 		// when a node is a control plane node we need the elastic IP
 		// to template out the kube-vip deployment
 		if machineScope.IsControlPlane() {
-			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
-				ctx,
-				machineScope.Cluster.Namespace,
-				machineScope.Cluster.Name,
-				machineScope.PacketCluster.Spec.ProjectID)
-			if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+			var controlPlaneEndpointAddress string
+			var cpemLBConfig string
+			var emlbID string
+			switch {
+			case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
+				controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
+					ctx,
+					machineScope.Cluster.Namespace,
+					machineScope.Cluster.Name,
+					machineScope.PacketCluster.Spec.ProjectID)
 				if len(controlPlaneEndpoint.Assignments) == 0 {
 					a := corev1.NodeAddress{
 						Type:    corev1.NodeExternalIP,
 						Address: controlPlaneEndpoint.GetAddress(),
 					}
 					addrs = append(addrs, a)
 				}
+				controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress()
+			case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID:
+				controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host
+				cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro
+				emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"]
 			}
-			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.GetAddress()
+			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpointAddress
+			createDeviceReq.CPEMLBConfig = cpemLBConfig
+			createDeviceReq.EMLBID = emlbID
 		}
-
 		dev, err = r.PacketClient.NewDevice(ctx, createDeviceReq)
 
 		switch {
@@ -413,7 +424,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	case infrav1.PacketResourceStatusRunning:
 		log.Info("Machine instance is active", "instance-id", machineScope.ProviderID())
 
-		if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+		switch {
+		case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
 			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
 				ctx,
 				machineScope.Cluster.Namespace,
@@ -428,6 +440,15 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 					return ctrl.Result{RequeueAfter: time.Second * 20}, nil
 				}
 			}
+		case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID:
+			if machineScope.IsControlPlane() {
+				// Create new EMLB object
+				lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro)
+
+				if err := lb.ReconcileVIPOrigin(ctx, machineScope, deviceAddr); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
 		}
 
 		machineScope.SetReady()

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/oauth2 v0.18.0
 	k8s.io/api v0.29.3
 	k8s.io/apimachinery v0.29.3
 	k8s.io/client-go v0.29.3
@@ -57,6 +59,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
@@ -73,7 +76,6 @@ require (
 	golang.org/x/crypto v0.23.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.20.0 // indirect