feat: Add Equinix Metal Load Balancer support

- Add internal package for interacting with Equinix Metal Load Balancer API - packetcluster_controller creates load balancer and listener port and stores their ids in packetCluster annotations. - packetmachine_controller creates an origin pool and origin port for each machine and stores their IDs in the packetMachine annotations. - CPEMLBConfig and EMLBID added to the packet cloud client package to be able to provide a config for the CPEM loadBalancer setting in the emlb templates. - Memory request for the Cluster API Provider Packet controller increased to 300Mi to avoid OOMing while debugging. - EMLB added as a valid VIPManager enum type. Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
kubernetes-sigs · Mar 12, 2024 · 0519aff · 0519aff
1 parent 45c53b4
commit 0519aff
Show file tree

Hide file tree

Showing 57 changed files with 10,436 additions and 97 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -126,7 +126,7 @@ jobs:
           REPO: ${{ github.event.repository.name }}
 
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: out/release/*

diff --git a/.golangci.yml b/.golangci.yml
@@ -164,7 +164,6 @@ linters-settings:
         alias: infraexpv1
   nolintlint:
     allow-unused: false
-    allow-leading-space: false
     require-specific: true
   revive:
     rules:
@@ -314,3 +313,7 @@ issues:
         - gocritic
       text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
       path: _test\.go
+    - linters:
+        - bodyclose
+      path: .*(internal)/emlb/emlb.go
+      text: "response body must be closed"
diff --git a/api/v1beta1/packetcluster_types.go b/api/v1beta1/packetcluster_types.go
@@ -26,7 +26,7 @@ const (
 	NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
 )
 
-// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip.
+// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer.
 type VIPManagerType string
 
 // PacketClusterSpec defines the desired state of PacketCluster.
@@ -46,9 +46,9 @@ type PacketClusterSpec struct {
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
 
-	// VIPManager represents whether this cluster uses CPEM or kube-vip to
+	// VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
 	// manage its vip for the api server IP
-	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
+	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB
 	// +kubebuilder:default:=CPEM
 	VIPManager VIPManagerType `json:"vipManager"`
 }

diff --git a/clusterctl-settings.json b/clusterctl-settings.json
@@ -2,7 +2,7 @@
     "name": "infrastructure-packet",
     "config": {
       "componentsFile": "infrastructure-components.yaml",
-      "nextVersion": "v0.6.99"
+      "nextVersion": "v0.8.99"
     }
   }
 
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml
@@ -75,10 +75,12 @@ spec:
               vipManager:
                 default: CPEM
                 description: VIPManager represents whether this cluster uses CPEM
-                  or kube-vip to manage its vip for the api server IP
+                  or kube-vip or Equinix Metal Load Balancer to manage its vip for
+                  the api server IP
                 enum:
                 - CPEM
                 - KUBE_VIP
+                - EMLB
                 type: string
             required:
             - projectID

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -42,10 +42,10 @@ spec:
               port: healthz
           resources:
             limits:
-              memory: 200Mi
+              memory: 300Mi
             requests:
               cpu: 100m
-              memory: 200Mi
+              memory: 300Mi
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

diff --git a/controllers/packetcluster_controller.go b/controllers/packetcluster_controller.go
@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 )
@@ -114,49 +115,63 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
 
 	packetCluster := clusterScope.PacketCluster
 
-	ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
 	switch {
-	case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
-		// Parse metro and facility from the cluster spec
-		var metro, facility string
-
-		facility = packetCluster.Spec.Facility
-		metro = packetCluster.Spec.Metro
-
-		// If both specified, metro takes precedence over facility
-		if metro != "" {
-			facility = ""
+	case packetCluster.Spec.VIPManager == "EMLB":
+		if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() {
+			// Create new EMLB object
+			lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro)
+
+			if err := lb.ReconcileLoadBalancer(ctx, clusterScope); err != nil {
+				log.Error(err, "Error Reconciling EMLB")
+				return err
+			}
 		}
-
-		// There is not an ElasticIP with the right tags, at this point we can create one
-		ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
-		if err != nil {
-			log.Error(err, "error reserving an ip")
+	case packetCluster.Spec.VIPManager == "KUBE_VIP":
+		log.Info("KUBE_VIP VIPManager Detected")
+		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
+			log.Error(err, "error enabling bgp for project")
 			return err
 		}
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ip.To4().String(),
-			Port: 6443,
-		}
-	case err != nil:
-		log.Error(err, "error getting cluster IP")
-		return err
-	default:
-		// If there is an ElasticIP with the right tag just use it again
-		clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
-			Host: ipReserv.GetAddress(),
-			Port: 6443,
-		}
 	}
 
-	if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
-		if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
-			log.Error(err, "error enabling bgp for project")
+	if packetCluster.Spec.VIPManager != "EMLB" {
+		ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
+		switch {
+		case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
+			// Parse metro and facility from the cluster spec
+			var metro, facility string
+
+			facility = packetCluster.Spec.Facility
+			metro = packetCluster.Spec.Metro
+
+			// If both specified, metro takes precedence over facility
+			if metro != "" {
+				facility = ""
+			}
+
+			// There is not an ElasticIP with the right tags, at this point we can create one
+			ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
+			if err != nil {
+				log.Error(err, "error reserving an ip")
+				return err
+			}
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ip.To4().String(),
+				Port: 6443,
+			}
+		case err != nil:
+			log.Error(err, "error getting cluster IP")
 			return err
+		default:
+			// If there is an ElasticIP with the right tag just use it again
+			packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
+				Host: ipReserv.GetAddress(),
+				Port: 6443,
+			}
 		}
 	}
 
-	clusterScope.PacketCluster.Status.Ready = true
+	packetCluster.Status.Ready = true
 	conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
 
 	return nil

diff --git a/controllers/packetmachine_controller.go b/controllers/packetmachine_controller.go
@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
 	packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
 	"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
 	clog "sigs.k8s.io/cluster-api/util/log"
@@ -348,23 +349,33 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 		// when a node is a control plane node we need the elastic IP
 		// to template out the kube-vip deployment
 		if machineScope.IsControlPlane() {
-			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
-				ctx,
-				machineScope.Cluster.Namespace,
-				machineScope.Cluster.Name,
-				machineScope.PacketCluster.Spec.ProjectID)
-			if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+			var controlPlaneEndpointAddress string
+			var cpemLBConfig string
+			var emlbID string
+			switch {
+			case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
+				controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
+					ctx,
+					machineScope.Cluster.Namespace,
+					machineScope.Cluster.Name,
+					machineScope.PacketCluster.Spec.ProjectID)
 				if len(controlPlaneEndpoint.Assignments) == 0 {
 					a := corev1.NodeAddress{
 						Type:    corev1.NodeExternalIP,
 						Address: controlPlaneEndpoint.GetAddress(),
 					}
 					addrs = append(addrs, a)
 				}
+				controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress()
+			case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
+				controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host
+				cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro
+				emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"]
 			}
-			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.GetAddress()
+			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpointAddress
+			createDeviceReq.CPEMLBConfig = cpemLBConfig
+			createDeviceReq.EMLBID = emlbID
 		}
-
 		dev, err = r.PacketClient.NewDevice(ctx, createDeviceReq)
 
 		switch {
@@ -413,7 +424,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	case infrav1.PacketResourceStatusRunning:
 		log.Info("Machine instance is active", "instance-id", machineScope.ProviderID())
 
-		if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+		switch {
+		case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
 			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
 				ctx,
 				machineScope.Cluster.Namespace,
@@ -428,6 +440,15 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 					return ctrl.Result{RequeueAfter: time.Second * 20}, nil
 				}
 			}
+		case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
+			if machineScope.IsControlPlane() {
+				// Create new EMLB object
+				lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro)
+
+				if err := lb.ReconcileVIPOrigin(ctx, machineScope, deviceAddr); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
 		}
 
 		machineScope.SetReady()

diff --git a/go.mod b/go.mod
@@ -3,15 +3,16 @@ module sigs.k8s.io/cluster-api-provider-packet
 go 1.20
 
 require (
-	github.com/equinix/equinix-sdk-go v0.32.0
+	github.com/equinix/equinix-sdk-go v0.35.0
 	github.com/onsi/gomega v1.30.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
-	k8s.io/api v0.28.6
-	k8s.io/apimachinery v0.28.6
-	k8s.io/client-go v0.28.6
-	k8s.io/component-base v0.28.6
+	golang.org/x/oauth2 v0.14.0
+	k8s.io/api v0.28.7
+	k8s.io/apimachinery v0.28.7
+	k8s.io/client-go v0.28.7
+	k8s.io/component-base v0.28.7
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/utils v0.0.0-20231127182322-b307cd553661
 	sigs.k8s.io/cluster-api v1.6.0
@@ -81,7 +82,6 @@ require (
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect

diff --git a/go.sum b/go.sum
@@ -38,8 +38,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
-github.com/equinix/equinix-sdk-go v0.32.0 h1:zUn0Em5FJe6f6bntftrDBpO9L+XhbpFMPuQ7RKEOgXM=
-github.com/equinix/equinix-sdk-go v0.32.0/go.mod h1:qnpdRzVftHFNaJFk1VSIrAOTLrIoeDrxzUr3l8ARyvQ=
+github.com/equinix/equinix-sdk-go v0.35.0 h1:p/uwA8QPBAuNnKGc3mQkwjw+6++qUadLNnKFQ8jw+wg=
+github.com/equinix/equinix-sdk-go v0.35.0/go.mod h1:hEb3XLaedz7xhl/dpPIS6eOIiXNPeqNiVoyDrT6paIg=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
@@ -165,7 +165,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -283,20 +283,20 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.28.6 h1:yy6u9CuIhmg55YvF/BavPBBXB+5QicB64njJXxVnzLo=
-k8s.io/api v0.28.6/go.mod h1:AM6Ys6g9MY3dl/XNaNfg/GePI0FT7WBGu8efU/lirAo=
+k8s.io/api v0.28.7 h1:YKIhBxjXKaxuxWJnwohV0aGjRA5l4IU0Eywf/q19AVI=
+k8s.io/api v0.28.7/go.mod h1:y4RbcjCCMff1930SG/TcP3AUKNfaJUgIeUp58e/2vyY=
 k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
 k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
-k8s.io/apimachinery v0.28.6 h1:RsTeR4z6S07srPg6XYrwXpTJVMXsjPXn0ODakMytSW0=
-k8s.io/apimachinery v0.28.6/go.mod h1:QFNX/kCl/EMT2WTSz8k4WLCv2XnkOLMaL8GAVRMdpsA=
+k8s.io/apimachinery v0.28.7 h1:2Z38/XRAOcpb+PonxmBEmjG7hBfmmr41xnr0XvpTnB4=
+k8s.io/apimachinery v0.28.7/go.mod h1:QFNX/kCl/EMT2WTSz8k4WLCv2XnkOLMaL8GAVRMdpsA=
 k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
 k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
-k8s.io/client-go v0.28.6 h1:Gge6ziyIdafRchfoBKcpaARuz7jfrK1R1azuwORIsQI=
-k8s.io/client-go v0.28.6/go.mod h1:+nu0Yp21Oeo/cBCsprNVXB2BfJTV51lFfe5tXl2rUL8=
+k8s.io/client-go v0.28.7 h1:3L6402+tjmOl8twX3fjUQ/wsYAkw6UlVNDVP+rF6YGA=
+k8s.io/client-go v0.28.7/go.mod h1:xIoEaDewZ+EwWOo1/F1t0IOKMPe1rwBZhLu9Es6y0tE=
 k8s.io/cluster-bootstrap v0.28.4 h1:4MKNy1Qd9QY7pl47rSMGIORF+tm3CUaqC1M8U9bjn4Q=
 k8s.io/cluster-bootstrap v0.28.4/go.mod h1:/c4ro/R4yf4EtJgFgFtvnHkbDOHwubeKJXh5R1c89Bc=
-k8s.io/component-base v0.28.6 h1:G4T8VrcQ7xZou3by/fY5NU5mfxOBlWaivS2lPrEltAo=
-k8s.io/component-base v0.28.6/go.mod h1:Dg62OOG3ALu2P4nAG00UdsuHoNLQJ5VsUZKQlLDcS+E=
+k8s.io/component-base v0.28.7 h1:Cq5aQ52N0CTaOMiary4rXzR4RoTP77Z3ll4qSg4qH7s=
+k8s.io/component-base v0.28.7/go.mod h1:RrtNBKrSuckksSQ3fV9PhwBSHO/ZbwJXM2Z0OPx+UJk=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kms v0.28.4 h1:PMgY/3CQTWP9eIKmNQiTgjLIZ0ns6O+voagzD2/4mSg=