Implement ipaddressallocation controller

build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml

@@ -54,12 +54,10 @@ spec:
               ip_address_block_visibility:
                 default: Private
                 description: IPAddressBlockVisibility specifies the visibility of
-                  the IPBlocks to allocate IP addresses. Can be External, Private
-                  or Project.
+                  the IPBlocks to allocate IP addresses. Can be External or Private.
                 enum:
                 - External
                 - Private
-                - Project
                 type: string
             type: object
           status:

build/yaml/samples/nsx_v1alpha1_ipaddressallocation.yaml

@@ -5,6 +5,4 @@ metadata:
   namespace: sc-a
 spec:
   ip_address_block_visibility: Private
-  allocation_size: 26
-status:
-  CIDR: 172.26.1.0/28
+  allocation_size: 32

cmd/main.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/ipaddressallocation"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	ippool2 "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ippool"
@@ -39,6 +40,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	ipaddressallocationservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ippool"
 	nodeservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/node"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/nsxserviceaccount"
@@ -147,6 +149,21 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp
 	}
 }
 
+func StartIPAddressAllocationController(mgr ctrl.Manager, ipAddressAllocationService *ipaddressallocationservice.IPAddressAllocationService, vpcService common.VPCServiceProvider) {
+	ipAddressAllocationReconciler := &ipaddressallocation.IPAddressAllocationReconciler{
+		Client:     mgr.GetClient(),
+		Scheme:     mgr.GetScheme(),
+		Service:    ipAddressAllocationService,
+		VPCService: vpcService,
+		Recorder:   mgr.GetEventRecorderFor("ipaddressallocation-controller"),
+	}
+
+	if err := ipAddressAllocationReconciler.SetupWithManager(mgr); err != nil {
+		log.Error(err, "failed to create ipaddressallocation controller")
+		os.Exit(1)
+	}
+}
+
 func main() {
 	log.Info("starting NSX Operator")
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -202,6 +219,10 @@ func main() {
 		if err != nil {
 			log.Error(err, "failed to initialize ippool commonService", "controller", "IPPool")
 		}
+		ipAddressAllocationService, err := ipaddressallocationservice.InitializeIPAddressAllocation(commonService, vpcService)
+		if err != nil {
+			log.Error(err, "failed to initialize ipaddressallocation commonService", "controller", "IPAddressAllocation")
+		}
 		subnetPortService, err := subnetportservice.InitializeSubnetPort(commonService)
 		if err != nil {
 			log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort")
@@ -238,6 +259,7 @@ func main() {
 		subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
 		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
 		StartIPPoolController(mgr, ipPoolService, vpcService)
+		StartIPAddressAllocationController(mgr, ipAddressAllocationService, vpcService)
 		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
 		service.StartServiceLbController(mgr, commonService)
 	}

pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go

@@ -42,8 +42,8 @@ type IPAddressAllocationList struct {
 
 // IPAddressAllocationSpec defines the desired state of IPAddressAllocation.
 type IPAddressAllocationSpec struct {
-	// IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project.
-	// +kubebuilder:validation:Enum=External;Private;Project
+	// IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private.
+	// +kubebuilder:validation:Enum=External;Private
 	// +kubebuilder:default=Private
 	// +optional
 	IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"`

pkg/apis/v1alpha1/ipaddressallocation_types.go

@@ -10,9 +10,8 @@ import (
 type IPAddressVisibility string
 
 const (
-	IPAddressVisibilityExternal = "External"
-	IPAddressVisibilityPrivate  = "Private"
-	IPAddressVisibilityProject  = "Project"
+	IPAddressVisibilityExternal = "EXTERNAL"
+	IPAddressVisibilityPrivate  = "PRIVATE"
 )
 
 // +genclient
@@ -42,8 +41,8 @@ type IPAddressAllocationList struct {
 
 // IPAddressAllocationSpec defines the desired state of IPAddressAllocation.
 type IPAddressAllocationSpec struct {
-	// IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project.
-	// +kubebuilder:validation:Enum=External;Private;Project
+	// IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private.
+	// +kubebuilder:validation:Enum=External;Private
 	// +kubebuilder:default=Private
 	// +optional
 	IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"`

pkg/controllers/common/types.go

@@ -8,20 +8,21 @@ import (
 )
 
 const (
-	MetricResTypeSecurityPolicy    = "securitypolicy"
-	MetricResTypeNetworkPolicy     = "networkpolicy"
-	MetricResTypeIPPool            = "ippool"
-	MetricResTypeNSXServiceAccount = "nsxserviceaccount"
-	MetricResTypeSubnetPort        = "subnetport"
-	MetricResTypeStaticRoute       = "staticroute"
-	MetricResTypeSubnet            = "subnet"
-	MetricResTypeSubnetSet         = "subnetset"
-	MetricResTypeNetworkInfo       = "networkinfo"
-	MetricResTypeNamespace         = "namespace"
-	MetricResTypePod               = "pod"
-	MetricResTypeNode              = "node"
-	MetricResTypeServiceLb         = "servicelb"
-	MaxConcurrentReconciles        = 8
+	MetricResTypeSecurityPolicy      = "securitypolicy"
+	MetricResTypeNetworkPolicy       = "networkpolicy"
+	MetricResTypeIPPool              = "ippool"
+	MetricResTypeIPAddressAllocation = "ipaddressallocation"
+	MetricResTypeNSXServiceAccount   = "nsxserviceaccount"
+	MetricResTypeSubnetPort          = "subnetport"
+	MetricResTypeStaticRoute         = "staticroute"
+	MetricResTypeSubnet              = "subnet"
+	MetricResTypeSubnetSet           = "subnetset"
+	MetricResTypeNetworkInfo         = "networkinfo"
+	MetricResTypeNamespace           = "namespace"
+	MetricResTypePod                 = "pod"
+	MetricResTypeNode                = "node"
+	MetricResTypeServiceLb           = "servicelb"
+	MaxConcurrentReconciles          = 8
 
 	LabelK8sMasterRole  = "node-role.kubernetes.io/master"
 	LabelK8sControlRole = "node-role.kubernetes.io/control-plane"

pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go

@@ -0,0 +1,217 @@
+/* Copyright © 2024 VMware, Inc. All Rights Reserved.
+   SPDX-License-Identifier: Apache-2.0 */
+
+package ipaddressallocation
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
+	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
+	servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation"
+)
+
+var (
+	log           = logger.Log
+	once          sync.Once
+	resultNormal  = common.ResultNormal
+	resultRequeue = common.ResultRequeue
+	MetricResType = common.MetricResTypeIPAddressAllocation
+)
+
+// IPAddressAllocationReconciler reconciles a IPAddressAllocation object
+type IPAddressAllocationReconciler struct {
+	client.Client
+	Scheme     *apimachineryruntime.Scheme
+	Service    *ipaddressallocation.IPAddressAllocationService
+	VPCService servicecommon.VPCServiceProvider
+	Recorder   record.EventRecorder
+}
+
+func deleteSuccess(r *IPAddressAllocationReconciler, _ *context.Context, o *v1alpha1.IPAddressAllocation) {
+	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulDelete, "IPAddressAllocation CR has been successfully deleted")
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType)
+}
+
+func deleteFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) {
+	r.setReadyStatusFalse(c, o, metav1.Now(), e)
+	r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailDelete, fmt.Sprintf("%v", *e))
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType)
+}
+
+func updateSuccess(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation) {
+	r.setReadyStatusTrue(c, o, metav1.Now())
+	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "IPAddressAllocation CR has been successfully updated")
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResType)
+}
+
+func updateFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) {
+	r.setReadyStatusFalse(c, o, metav1.Now(), e)
+	r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailUpdate, fmt.Sprintf("%v", *e))
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResType)
+}
+
+func (r *IPAddressAllocationReconciler) setReadyStatusFalse(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time, err *error) {
+	conditions := []v1alpha1.Condition{
+		{
+			Type:    v1alpha1.Ready,
+			Status:  v1.ConditionFalse,
+			Message: "NSX IPAddressAllocation could not be created or updated",
+			Reason: fmt.Sprintf(
+				"error occurred while processing the IPAddressAllocation CR. Error: %v",
+				*err,
+			),
+			LastTransitionTime: transitionTime,
+		},
+	}
+	ipaddressallocation.Status.Conditions = conditions
+	e := r.Client.Status().Update(*ctx, ipaddressallocation)
+	if e != nil {
+		log.Error(e, "unable to update IPAddressAllocation status", "IPAddressAllocation", ipaddressallocation)
+	}
+}
+
+func (r *IPAddressAllocationReconciler) setReadyStatusTrue(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time) {
+	conditions := []v1alpha1.Condition{
+		{
+			Type:               v1alpha1.Ready,
+			Status:             v1.ConditionTrue,
+			Message:            "NSX IPAddressAllocation has been successfully created/updated",
+			Reason:             "",
+			LastTransitionTime: transitionTime,
+		},
+	}
+	ipaddressallocation.Status.Conditions = conditions
+	e := r.Client.Status().Update(*ctx, ipaddressallocation)
+	if e != nil {
+		log.Error(e, "unable to update IPAddressAllocation status", "IPAddressAllocation", ipaddressallocation)
+	}
+}
+
+func (r *IPAddressAllocationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	once.Do(func() { go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, r.collectGarbage) })
+	obj := &v1alpha1.IPAddressAllocation{}
+	log.Info("reconciling IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName)
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType)
+	if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil {
+		log.Error(err, "unable to fetch IPAddressAllocation CR", "req", req.NamespacedName)
+		return resultNormal, client.IgnoreNotFound(err)
+	}
+	if obj.ObjectMeta.DeletionTimestamp.IsZero() {
+		return r.handleUpdate(ctx, req, obj)
+	}
+	return r.handleDeletion(ctx, req, obj)
+}
+
+func (r *IPAddressAllocationReconciler) handleUpdate(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) {
+	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType)
+	if !controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) {
+		controllerutil.AddFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName)
+		if err := r.Client.Update(ctx, obj); err != nil {
+			log.Error(err, "add finalizer", "IPAddressAllocation", req.NamespacedName)
+			updateFail(r, &ctx, obj, &err)
+			return resultRequeue, err
+		}
+		log.V(1).Info("added finalizer on IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName)
+	}
+
+	updated, err := r.Service.CreateOrUpdateIPAddressAllocation(obj)
+	if err != nil {
+		updateFail(r, &ctx, obj, &err)
+		return resultRequeue, err
+	}
+	if updated {
+		updateSuccess(r, &ctx, obj)
+	}
+	return resultNormal, nil
+}
+
+func (r *IPAddressAllocationReconciler) handleDeletion(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) {
+	if controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) {
+		metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType)
+		if err := r.Service.DeleteIPAddressAllocation(obj); err != nil {
+			log.Error(err, "deletion failed, would retry exponentially", "IPAddressAllocation", req.NamespacedName)
+			deleteFail(r, &ctx, obj, &err)
+			return resultRequeue, err
+		}
+		controllerutil.RemoveFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName)
+		if err := r.Client.Update(ctx, obj); err != nil {
+			log.Error(err, "deletion failed, would retry exponentially", "IPAddressAllocation", req.NamespacedName)
+			deleteFail(r, &ctx, obj, &err)
+			return resultRequeue, err
+		}
+		log.V(1).Info("removed finalizer on IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName)
+		deleteSuccess(r, &ctx, obj)
+		log.Info("successfully deleted IPAddressAllocation CR and all subnets", "IPAddressAllocation", obj)
+	} else {
+		// only print a message because it's not a normal case
+		log.Info("IPAddressAllocation CR is being deleted but its finalizers cannot be recognized", "IPAddressAllocation", req.NamespacedName)
+	}
+	return resultNormal, nil
+}
+
+func (r *IPAddressAllocationReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.IPAddressAllocation{}).
+		WithEventFilter(predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				// Ignore updates to CR status in which case metadata.Generation does not change
+				return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration()
+			},
+			DeleteFunc: func(e event.DeleteEvent) bool {
+				// Suppress Delete events to avoid filtering them out in the Reconcile function
+				return false
+			},
+		}).
+		WithOptions(
+			controller.Options{
+				MaxConcurrentReconciles: common.NumReconcile(),
+			}).
+		Complete(r)
+}
+
+func (r *IPAddressAllocationReconciler) collectGarbage(ctx context.Context) {
+	log.Info("IPAddressAllocation garbage collector started")
+	ipAddressAllocationSet := r.Service.ListIPAddressAllocationID()
+	if len(ipAddressAllocationSet) == 0 {
+		return
+	}
+
+	ipAddressAllocationList := &v1alpha1.IPAddressAllocationList{}
+	if err := r.Client.List(ctx, ipAddressAllocationList); err != nil {
+		log.Error(err, "failed to list IPAddressAllocation CR")
+		return
+	}
+	CRIPAddressAllocationSet := sets.New[string]()
+	for _, ipa := range ipAddressAllocationList.Items {
+		CRIPAddressAllocationSet.Insert(string(ipa.UID))
+	}
+
+	log.V(2).Info("IPAddressAllocation garbage collector", "nsxIPAddressAllocationSet", ipAddressAllocationSet, "CRIPAddressAllocationSet", CRIPAddressAllocationSet)
+
+	diffSet := ipAddressAllocationSet.Difference(CRIPAddressAllocationSet)
+	for elem := range diffSet {
+		log.Info("GC collected nsx IPAddressAllocation", "UID", elem)
+		if err := r.Service.DeleteIPAddressAllocation(types.UID(elem)); err != nil {
+			log.Error(err, "failed to delete nsx IPAddressAllocation", "UID", elem)
+		}
+	}
+}