Merge pull request #320 from pluralsh/kube-vip

✨ Add support for kube-vip

Makefile

@@ -275,6 +275,7 @@ generate: ## Generate code
 
 .PHONY: generate-templates
 generate-templates: $(KUSTOMIZE) ## Generate cluster templates
+	$(KUSTOMIZE) build templates/experimental-kube-vip --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip.yaml
 	$(KUSTOMIZE) build templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-crs-cni.yaml
 	$(KUSTOMIZE) build templates/addons/calico > templates/addons/calico.yaml
 

api/v1alpha3/packetcluster_types.go

@@ -21,6 +21,9 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 )
 
+// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip
+type VIPManagerType string
+
 // PacketClusterSpec defines the desired state of PacketCluster
 type PacketClusterSpec struct {
 	// ProjectID represents the Packet Project where this cluster will be placed into
@@ -32,6 +35,12 @@ type PacketClusterSpec struct {
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
+
+	// VIPManager represents whether this cluster uses CPEM or kube-vip to
+	// manage its vip for the api server IP
+	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
+	// +kubebuilder:default:=CPEM
+	VIPManager VIPManagerType `json:"vipManager"`
 }
 
 // PacketClusterStatus defines the observed state of PacketCluster

api/v1beta1/packetcluster_types.go

@@ -26,6 +26,9 @@ const (
 	NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
 )
 
+// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip
+type VIPManagerType string
+
 // PacketClusterSpec defines the desired state of PacketCluster
 type PacketClusterSpec struct {
 	// ProjectID represents the Packet Project where this cluster will be placed into
@@ -37,6 +40,12 @@ type PacketClusterSpec struct {
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
+
+	// VIPManager represents whether this cluster uses CPEM or kube-vip to
+	// manage its vip for the api server IP
+	// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
+	// +kubebuilder:default:=CPEM
+	VIPManager VIPManagerType `json:"vipManager"`
 }
 
 // PacketClusterStatus defines the observed state of PacketCluster

api/v1beta1/packetcluster_webhook.go

@@ -71,6 +71,13 @@ func (c *PacketCluster) ValidateUpdate(oldRaw runtime.Object) error {
 		)
 	}
 
+	if !reflect.DeepEqual(c.Spec.VIPManager, old.Spec.VIPManager) {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "VIPManager"),
+				c.Spec.VIPManager, "field is immutable"),
+		)
+	}
+
 	if len(allErrs) == 0 {
 		return nil
 	}

config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml

@@ -68,8 +68,17 @@ spec:
                 description: ProjectID represents the Packet Project where this cluster
                   will be placed into
                 type: string
+              vipManager:
+                default: CPEM
+                description: VIPManager represents whether this cluster uses CPEM
+                  or kube-vip to manage its vip for the api server IP
+                enum:
+                - CPEM
+                - KUBE_VIP
+                type: string
             required:
             - projectID
+            - vipManager
             type: object
           status:
             description: PacketClusterStatus defines the observed state of PacketCluster
@@ -134,8 +143,17 @@ spec:
                 description: ProjectID represents the Packet Project where this cluster
                   will be placed into
                 type: string
+              vipManager:
+                default: CPEM
+                description: VIPManager represents whether this cluster uses CPEM
+                  or kube-vip to manage its vip for the api server IP
+                enum:
+                - CPEM
+                - KUBE_VIP
+                type: string
             required:
             - projectID
+            - vipManager
             type: object
           status:
             description: PacketClusterStatus defines the observed state of PacketCluster

config/default/kustomization.yaml

@@ -26,7 +26,7 @@ patchesStrategicMerge:
 - manager_pull_policy.yaml
 - manager_webhook_patch.yaml
 - webhookcainjection_patch.yaml
-- manager_credentials_patch.yaml
+- manager_credentials_config_patch.yaml
 
 vars:
   - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR

config/default/kustomizeconfig.yaml

@@ -2,3 +2,5 @@
 varReference:
 - kind: Deployment
   path: spec/template/spec/volumes/secret/secretName
+- kind: Deployment
+  path: spec/template/spec/volumes/configMap/configMapName

controllers/packetcluster_controller.go

@@ -137,6 +137,13 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
 		}
 	}
 
+	if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
+		if err := r.PacketClient.EnableProjectBGP(packetCluster.Spec.ProjectID); err != nil {
+			log.Error(err, "error enabling bgp for project")
+			return ctrl.Result{}, err
+		}
+	}
+
 	clusterScope.PacketCluster.Status.Ready = true
 	conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
 

controllers/packetmachine_controller.go

@@ -222,7 +222,7 @@ func (r *PacketMachineReconciler) PacketClusterToPacketMachines(ctx context.Cont
 	}
 }
 
-func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) { //nolint:gocyclo
+func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) { //nolint:gocyclo,maintidx
 	log := ctrl.LoggerFrom(ctx, "machine", machineScope.Machine.Name, "cluster", machineScope.Cluster.Name)
 	log.Info("Reconciling PacketMachine")
 
@@ -316,21 +316,21 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 			ExtraTags:    packet.DefaultCreateTags(machineScope.Namespace(), machineScope.Machine.Name, machineScope.Cluster.Name),
 		}
 
-		// TODO: see if this can be removed with kube-vip in place
-		// when the node is a control plan we should check if the elastic ip
-		// for this cluster is not assigned. If it is free we can prepare the
-		// current node to use it.
+		// when a node is a control plane node we need the elastic IP
+		// to template out the kube-vip deployment
 		if machineScope.IsControlPlane() {
 			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
 				machineScope.Cluster.Namespace,
 				machineScope.Cluster.Name,
 				machineScope.PacketCluster.Spec.ProjectID)
-			if len(controlPlaneEndpoint.Assignments) == 0 {
-				a := corev1.NodeAddress{
-					Type:    corev1.NodeExternalIP,
-					Address: controlPlaneEndpoint.Address,
+			if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+				if len(controlPlaneEndpoint.Assignments) == 0 {
+					a := corev1.NodeAddress{
+						Type:    corev1.NodeExternalIP,
+						Address: controlPlaneEndpoint.Address,
+					}
+					addrs = append(addrs, a)
 				}
-				addrs = append(addrs, a)
 			}
 			createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.Address
 		}
@@ -362,6 +362,13 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	machineScope.SetProviderID(dev.ID)
 	machineScope.SetInstanceStatus(infrav1.PacketResourceStatus(dev.State))
 
+	if machineScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
+		if err := r.PacketClient.EnsureNodeBGPEnabled(dev.ID); err != nil {
+			// Do not treat an error enabling bgp on machine as fatal
+			return ctrl.Result{RequeueAfter: time.Second * 20}, fmt.Errorf("failed to enable bpg on machine %s: %w", machineScope.Name(), err)
+		}
+	}
+
 	deviceAddr := r.PacketClient.GetDeviceAddresses(dev)
 	machineScope.SetAddresses(append(addrs, deviceAddr...))
 
@@ -376,22 +383,21 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 	case infrav1.PacketResourceStatusRunning:
 		log.Info("Machine instance is active", "instance-id", machineScope.GetInstanceID())
 
-		// TODO: see if this can be removed with kube-vip in place
-		// This logic is here because an elastic ip can be assigned only an
-		// active node. It needs to be a control plane and the IP should not be
-		// assigned to anything at this point.
-		controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
-			machineScope.Cluster.Namespace,
-			machineScope.Cluster.Name,
-			machineScope.PacketCluster.Spec.ProjectID)
-		if len(controlPlaneEndpoint.Assignments) == 0 && machineScope.IsControlPlane() {
-			if _, _, err := r.PacketClient.DeviceIPs.Assign(dev.ID, &packngo.AddressStruct{
-				Address: controlPlaneEndpoint.Address,
-			}); err != nil {
-				log.Error(err, "err assigining elastic ip to control plane. retrying...")
-				return ctrl.Result{RequeueAfter: time.Second * 20}, nil
+		if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
+			controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
+				machineScope.Cluster.Namespace,
+				machineScope.Cluster.Name,
+				machineScope.PacketCluster.Spec.ProjectID)
+			if len(controlPlaneEndpoint.Assignments) == 0 && machineScope.IsControlPlane() {
+				if _, _, err := r.PacketClient.DeviceIPs.Assign(dev.ID, &packngo.AddressStruct{
+					Address: controlPlaneEndpoint.Address,
+				}); err != nil {
+					log.Error(err, "err assigining elastic ip to control plane. retrying...")
+					return ctrl.Result{RequeueAfter: time.Second * 20}, nil
+				}
 			}
 		}
+
 		machineScope.SetReady()
 		conditions.MarkTrue(machineScope.PacketMachine, infrav1.DeviceReadyCondition)
 

pkg/cloud/packet/client.go

@@ -23,6 +23,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"text/template"
 
@@ -38,6 +39,9 @@ const (
 	apiTokenVarName = "PACKET_API_KEY" //nolint:gosec
 	clientName      = "CAPP-v1beta1"
 	ipxeOS          = "custom_ipxe"
+	envVarLocalASN  = "METAL_LOCAL_ASN"
+	envVarBGPPass   = "METAL_BGP_PASS" //nolint:gosec
+	DefaultLocalASN = 65000
 )
 
 var (
@@ -230,6 +234,70 @@ func (p *Client) CreateIP(namespace, clusterName, projectID, facility string) (n
 	return ip, nil
 }
 
+// enableBGP enable bgp on the project
+func (p *Client) EnableProjectBGP(projectID string) error {
+	// first check if it is enabled before trying to create it
+	bgpConfig, _, err := p.BGPConfig.Get(projectID, &packngo.GetOptions{})
+	// if we already have a config, just return
+	// we need some extra handling logic because the API always returns 200, even if
+	// not BGP config is in place.
+	// We treat it as valid config already exists only if ALL of the above is true:
+	// - no error
+	// - bgpConfig struct exists
+	// - bgpConfig struct has non-blank ID
+	// - bgpConfig struct does not have Status=="disabled"
+	if err != nil {
+		return err
+	} else if bgpConfig != nil && bgpConfig.ID != "" && strings.ToLower(bgpConfig.Status) != "disabled" {
+		return nil
+	}
+
+	// get the local ASN
+	localASN := os.Getenv(envVarLocalASN)
+	var outLocalASN int
+	switch {
+	case localASN != "":
+		localASNNo, err := strconv.Atoi(localASN)
+		if err != nil {
+			return fmt.Errorf("env var %s must be a number, was %s: %w", envVarLocalASN, localASN, err)
+		}
+		outLocalASN = localASNNo
+	default:
+		outLocalASN = DefaultLocalASN
+	}
+
+	var outBGPPass string
+	bgpPass := os.Getenv(envVarBGPPass)
+	if bgpPass != "" {
+		outBGPPass = bgpPass
+	}
+
+	// we did not have a valid one, so create it
+	req := packngo.CreateBGPConfigRequest{
+		Asn:            outLocalASN,
+		Md5:            outBGPPass,
+		DeploymentType: "local",
+		UseCase:        "kubernetes-load-balancer",
+	}
+	_, err = p.BGPConfig.Create(projectID, req)
+	return err
+}
+
+// ensureNodeBGPEnabled check if the node has bgp enabled, and set it if it does not
+func (p *Client) EnsureNodeBGPEnabled(id string) error {
+	// fortunately, this is idempotent, so just create
+	req := packngo.CreateBGPSessionRequest{
+		AddressFamily: "ipv4",
+	}
+	_, response, err := p.BGPSessions.Create(id, req)
+	// if we already had one, then we can ignore the error
+	// this really should be a 409, but 422 is what is returned
+	if response.StatusCode == 422 && strings.Contains(fmt.Sprintf("%s", err), "already has session") {
+		err = nil
+	}
+	return err
+}
+
 func (p *Client) GetIPByClusterIdentifier(namespace, name, projectID string) (packngo.IPAddressReservation, error) {
 	var err error
 	var reservedIP packngo.IPAddressReservation

templates/addons/calico.yaml

@@ -2497,7 +2497,7 @@ spec:
           value: node
         - name: DATASTORE_TYPE
           value: kubernetes
-        image: quay.io/calico/kube-controllers:v3.20.4
+        image: quay.io/calico/kube-controllers:v3.20.5
         livenessProbe:
           exec:
             command:
@@ -2603,7 +2603,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/node:v3.20.4
+        image: quay.io/calico/node:v3.20.5
         lifecycle:
           preStop:
             exec:
@@ -2677,7 +2677,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/cni:v3.20.4
+        image: quay.io/calico/cni:v3.20.5
         name: upgrade-ipam
         securityContext:
           privileged: true
@@ -2711,7 +2711,7 @@ spec:
         - configMapRef:
             name: kubernetes-services-endpoint
             optional: true
-        image: quay.io/calico/cni:v3.20.4
+        image: quay.io/calico/cni:v3.20.5
         name: install-cni
         securityContext:
           privileged: true
@@ -2720,7 +2720,7 @@ spec:
           name: cni-bin-dir
         - mountPath: /host/etc/cni/net.d
           name: cni-net-dir
-      - image: quay.io/calico/pod2daemon-flexvol:v3.20.4
+      - image: quay.io/calico/pod2daemon-flexvol:v3.20.5
         name: flexvol-driver
         securityContext:
           privileged: true