From c46325e6fe2ab77d2f60bd392f2fb8ae3d92b5dc Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Thu, 16 May 2024 11:09:22 +0800 Subject: [PATCH] Implement ipaddressallocation controller Signed-off-by: Xie Zheng --- .../nsx.vmware.com_ipaddressallocations.yaml | 4 +- .../nsx_v1alpha1_ipaddressallocation.yaml | 4 +- cmd/main.go | 22 ++ .../v1alpha1/ipaddressallocation_types.go | 4 +- .../v1alpha1/ipaddressallocation_types.go | 9 +- pkg/controllers/common/types.go | 29 +- .../ipaddressallocation_controller.go | 217 +++++++++++++ .../ipaddressallocation_controller_test.go | 301 ++++++++++++++++++ .../nsxserviceaccount_controller_test.go | 3 +- pkg/nsx/client.go | 41 +-- pkg/nsx/services/common/types.go | 50 +-- .../services/ipaddressallocation/builder.go | 49 +++ .../ipaddressallocation/builder_test.go | 115 +++++++ .../services/ipaddressallocation/compare.go | 32 ++ .../ipaddressallocation/compare_test.go | 52 +++ .../ipaddressallocation.go | 199 ++++++++++++ pkg/nsx/services/ipaddressallocation/store.go | 90 ++++++ .../ipaddressallocation/store_test.go | 98 ++++++ pkg/util/utils.go | 4 + 19 files changed, 1253 insertions(+), 70 deletions(-) create mode 100644 pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go create mode 100644 pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/builder.go create mode 100644 pkg/nsx/services/ipaddressallocation/builder_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/compare.go create mode 100644 pkg/nsx/services/ipaddressallocation/compare_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/ipaddressallocation.go create mode 100644 pkg/nsx/services/ipaddressallocation/store.go create mode 100644 pkg/nsx/services/ipaddressallocation/store_test.go diff --git a/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml b/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml index 476fa42d6..8b0bff706 100644 --- a/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml +++ b/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml @@ -54,12 +54,10 @@ spec: ip_address_block_visibility: default: Private description: IPAddressBlockVisibility specifies the visibility of - the IPBlocks to allocate IP addresses. Can be External, Private - or Project. + the IPBlocks to allocate IP addresses. Can be External or Private. enum: - External - Private - - Project type: string type: object status: diff --git a/build/yaml/samples/nsx_v1alpha1_ipaddressallocation.yaml b/build/yaml/samples/nsx_v1alpha1_ipaddressallocation.yaml index c56cebd7f..a1b7bf29a 100644 --- a/build/yaml/samples/nsx_v1alpha1_ipaddressallocation.yaml +++ b/build/yaml/samples/nsx_v1alpha1_ipaddressallocation.yaml @@ -5,6 +5,4 @@ metadata: namespace: sc-a spec: ip_address_block_visibility: Private - allocation_size: 26 -status: - CIDR: 172.26.1.0/28 \ No newline at end of file + allocation_size: 32 \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 1abec4603..8be859b89 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2" + "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ipaddressallocation" "github.com/vmware-tanzu/nsx-operator/pkg/config" ippool2 "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ippool" @@ -39,6 +40,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/metrics" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + ipaddressallocationservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ippool" nodeservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/node" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/nsxserviceaccount" @@ -147,6 +149,21 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp } } +func StartIPAddressAllocationController(mgr ctrl.Manager, ipAddressAllocationService *ipaddressallocationservice.IPAddressAllocationService, vpcService common.VPCServiceProvider) { + ipAddressAllocationReconciler := &ipaddressallocation.IPAddressAllocationReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Service: ipAddressAllocationService, + VPCService: vpcService, + Recorder: mgr.GetEventRecorderFor("ipaddressallocation-controller"), + } + + if err := ipAddressAllocationReconciler.SetupWithManager(mgr); err != nil { + log.Error(err, "failed to create ipaddressallocation controller") + os.Exit(1) + } +} + func main() { log.Info("starting NSX Operator") mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -202,6 +219,10 @@ func main() { if err != nil { log.Error(err, "failed to initialize ippool commonService", "controller", "IPPool") } + ipAddressAllocationService, err := ipaddressallocationservice.InitializeIPAddressAllocation(commonService, vpcService) + if err != nil { + log.Error(err, "failed to initialize ipaddressallocation commonService", "controller", "IPAddressAllocation") + } subnetPortService, err := subnetportservice.InitializeSubnetPort(commonService) if err != nil { log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort") @@ -238,6 +259,7 @@ func main() { subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService) pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService) StartIPPoolController(mgr, ipPoolService, vpcService) + StartIPAddressAllocationController(mgr, ipAddressAllocationService, vpcService) networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService) service.StartServiceLbController(mgr, commonService) } diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go b/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go index 6a50925c5..5e6ff5905 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go @@ -42,8 +42,8 @@ type IPAddressAllocationList struct { // IPAddressAllocationSpec defines the desired state of IPAddressAllocation. type IPAddressAllocationSpec struct { - // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project. - // +kubebuilder:validation:Enum=External;Private;Project + // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private. + // +kubebuilder:validation:Enum=External;Private // +kubebuilder:default=Private // +optional IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"` diff --git a/pkg/apis/v1alpha1/ipaddressallocation_types.go b/pkg/apis/v1alpha1/ipaddressallocation_types.go index 6a50925c5..4c50d0e2d 100644 --- a/pkg/apis/v1alpha1/ipaddressallocation_types.go +++ b/pkg/apis/v1alpha1/ipaddressallocation_types.go @@ -10,9 +10,8 @@ import ( type IPAddressVisibility string const ( - IPAddressVisibilityExternal = "External" - IPAddressVisibilityPrivate = "Private" - IPAddressVisibilityProject = "Project" + IPAddressVisibilityExternal = "EXTERNAL" + IPAddressVisibilityPrivate = "PRIVATE" ) // +genclient @@ -42,8 +41,8 @@ type IPAddressAllocationList struct { // IPAddressAllocationSpec defines the desired state of IPAddressAllocation. type IPAddressAllocationSpec struct { - // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project. - // +kubebuilder:validation:Enum=External;Private;Project + // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private. + // +kubebuilder:validation:Enum=External;Private // +kubebuilder:default=Private // +optional IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"` diff --git a/pkg/controllers/common/types.go b/pkg/controllers/common/types.go index e5c8a4392..f0a9c6158 100644 --- a/pkg/controllers/common/types.go +++ b/pkg/controllers/common/types.go @@ -8,20 +8,21 @@ import ( ) const ( - MetricResTypeSecurityPolicy = "securitypolicy" - MetricResTypeNetworkPolicy = "networkpolicy" - MetricResTypeIPPool = "ippool" - MetricResTypeNSXServiceAccount = "nsxserviceaccount" - MetricResTypeSubnetPort = "subnetport" - MetricResTypeStaticRoute = "staticroute" - MetricResTypeSubnet = "subnet" - MetricResTypeSubnetSet = "subnetset" - MetricResTypeNetworkInfo = "networkinfo" - MetricResTypeNamespace = "namespace" - MetricResTypePod = "pod" - MetricResTypeNode = "node" - MetricResTypeServiceLb = "servicelb" - MaxConcurrentReconciles = 8 + MetricResTypeSecurityPolicy = "securitypolicy" + MetricResTypeNetworkPolicy = "networkpolicy" + MetricResTypeIPPool = "ippool" + MetricResTypeIPAddressAllocation = "ipaddressallocation" + MetricResTypeNSXServiceAccount = "nsxserviceaccount" + MetricResTypeSubnetPort = "subnetport" + MetricResTypeStaticRoute = "staticroute" + MetricResTypeSubnet = "subnet" + MetricResTypeSubnetSet = "subnetset" + MetricResTypeNetworkInfo = "networkinfo" + MetricResTypeNamespace = "namespace" + MetricResTypePod = "pod" + MetricResTypeNode = "node" + MetricResTypeServiceLb = "servicelb" + MaxConcurrentReconciles = 8 LabelK8sMasterRole = "node-role.kubernetes.io/master" LabelK8sControlRole = "node-role.kubernetes.io/control-plane" diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go new file mode 100644 index 000000000..031765295 --- /dev/null +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go @@ -0,0 +1,217 @@ +/* Copyright © 2024 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package ipaddressallocation + +import ( + "context" + "fmt" + "sync" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "github.com/vmware-tanzu/nsx-operator/pkg/metrics" + servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" +) + +var ( + log = logger.Log + once sync.Once + resultNormal = common.ResultNormal + resultRequeue = common.ResultRequeue + MetricResType = common.MetricResTypeIPAddressAllocation +) + +// IPAddressAllocationReconciler reconciles a IPAddressAllocation object +type IPAddressAllocationReconciler struct { + client.Client + Scheme *apimachineryruntime.Scheme + Service *ipaddressallocation.IPAddressAllocationService + VPCService servicecommon.VPCServiceProvider + Recorder record.EventRecorder +} + +func deleteSuccess(r *IPAddressAllocationReconciler, _ *context.Context, o *v1alpha1.IPAddressAllocation) { + r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulDelete, "IPAddressAllocation CR has been successfully deleted") + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) +} + +func deleteFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) { + r.setReadyStatusFalse(c, o, metav1.Now(), e) + r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailDelete, fmt.Sprintf("%v", *e)) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) +} + +func updateSuccess(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation) { + r.setReadyStatusTrue(c, o, metav1.Now()) + r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "IPAddressAllocation CR has been successfully updated") + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResType) +} + +func updateFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) { + r.setReadyStatusFalse(c, o, metav1.Now(), e) + r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailUpdate, fmt.Sprintf("%v", *e)) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResType) +} + +func (r *IPAddressAllocationReconciler) setReadyStatusFalse(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time, err *error) { + conditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: "NSX IPAddressAllocation could not be created or updated", + Reason: fmt.Sprintf( + "error occurred while processing the IPAddressAllocation CR. Error: %v", + *err, + ), + LastTransitionTime: transitionTime, + }, + } + ipaddressallocation.Status.Conditions = conditions + e := r.Client.Status().Update(*ctx, ipaddressallocation) + if e != nil { + log.Error(e, "unable to update IPAddressAllocation status", "IPAddressAllocation", ipaddressallocation) + } +} + +func (r *IPAddressAllocationReconciler) setReadyStatusTrue(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time) { + conditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionTrue, + Message: "NSX IPAddressAllocation has been successfully created/updated", + Reason: "", + LastTransitionTime: transitionTime, + }, + } + ipaddressallocation.Status.Conditions = conditions + e := r.Client.Status().Update(*ctx, ipaddressallocation) + if e != nil { + log.Error(e, "unable to update IPAddressAllocation status", "IPAddressAllocation", ipaddressallocation) + } +} + +func (r *IPAddressAllocationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + once.Do(func() { go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, r.collectGarbage) }) + obj := &v1alpha1.IPAddressAllocation{} + log.Info("reconciling IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) + if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { + log.Error(err, "unable to fetch IPAddressAllocation CR", "req", req.NamespacedName) + return resultNormal, client.IgnoreNotFound(err) + } + if obj.ObjectMeta.DeletionTimestamp.IsZero() { + return r.handleUpdate(ctx, req, obj) + } + return r.handleDeletion(ctx, req, obj) +} + +func (r *IPAddressAllocationReconciler) handleUpdate(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) + if !controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) { + controllerutil.AddFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) + if err := r.Client.Update(ctx, obj); err != nil { + log.Error(err, "add finalizer", "IPAddressAllocation", req.NamespacedName) + updateFail(r, &ctx, obj, &err) + return resultRequeue, err + } + log.V(1).Info("added finalizer on IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName) + } + + updated, err := r.Service.CreateOrUpdateIPAddressAllocation(obj) + if err != nil { + updateFail(r, &ctx, obj, &err) + return resultRequeue, err + } + if updated { + updateSuccess(r, &ctx, obj) + } + return resultNormal, nil +} + +func (r *IPAddressAllocationReconciler) handleDeletion(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) { + if controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + if err := r.Service.DeleteIPAddressAllocation(obj); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "IPAddressAllocation", req.NamespacedName) + deleteFail(r, &ctx, obj, &err) + return resultRequeue, err + } + controllerutil.RemoveFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) + if err := r.Client.Update(ctx, obj); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "IPAddressAllocation", req.NamespacedName) + deleteFail(r, &ctx, obj, &err) + return resultRequeue, err + } + log.V(1).Info("removed finalizer on IPAddressAllocation CR", "IPAddressAllocation", req.NamespacedName) + deleteSuccess(r, &ctx, obj) + log.Info("successfully deleted IPAddressAllocation CR and all subnets", "IPAddressAllocation", obj) + } else { + // only print a message because it's not a normal case + log.Info("IPAddressAllocation CR is being deleted but its finalizers cannot be recognized", "IPAddressAllocation", req.NamespacedName) + } + return resultNormal, nil +} + +func (r *IPAddressAllocationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.IPAddressAllocation{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Ignore updates to CR status in which case metadata.Generation does not change + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Suppress Delete events to avoid filtering them out in the Reconcile function + return false + }, + }). + WithOptions( + controller.Options{ + MaxConcurrentReconciles: common.NumReconcile(), + }). + Complete(r) +} + +func (r *IPAddressAllocationReconciler) collectGarbage(ctx context.Context) { + log.Info("IPAddressAllocation garbage collector started") + ipAddressAllocationSet := r.Service.ListIPAddressAllocationID() + if len(ipAddressAllocationSet) == 0 { + return + } + + ipAddressAllocationList := &v1alpha1.IPAddressAllocationList{} + if err := r.Client.List(ctx, ipAddressAllocationList); err != nil { + log.Error(err, "failed to list IPAddressAllocation CR") + return + } + CRIPAddressAllocationSet := sets.New[string]() + for _, ipa := range ipAddressAllocationList.Items { + CRIPAddressAllocationSet.Insert(string(ipa.UID)) + } + + log.V(2).Info("IPAddressAllocation garbage collector", "nsxIPAddressAllocationSet", ipAddressAllocationSet, "CRIPAddressAllocationSet", CRIPAddressAllocationSet) + + diffSet := ipAddressAllocationSet.Difference(CRIPAddressAllocationSet) + for elem := range diffSet { + log.Info("GC collected nsx IPAddressAllocation", "UID", elem) + if err := r.Service.DeleteIPAddressAllocation(types.UID(elem)); err != nil { + log.Error(err, "failed to delete nsx IPAddressAllocation", "UID", elem) + } + } +} diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go new file mode 100644 index 000000000..6a724e2a7 --- /dev/null +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go @@ -0,0 +1,301 @@ +/* Copyright © 2021 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package ipaddressallocation + +import ( + "context" + "errors" + "reflect" + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + _ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" +) + +func NewFakeIPAddressAllocationReconciler() *IPAddressAllocationReconciler { + return &IPAddressAllocationReconciler{ + Client: fake.NewClientBuilder().Build(), + Scheme: fake.NewClientBuilder().Build().Scheme(), + Service: nil, + } +} + +func TestIPAddressAllocationController_setReadyStatusTrue(t *testing.T) { + r := NewFakeIPAddressAllocationReconciler() + ctx := context.TODO() + dummyIPAddressAllocation := &v1alpha1.IPAddressAllocation{} + transitionTime := metav1.Now() + + // Case: Static Route CRD creation fails + newConditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionTrue, + Message: "NSX IPAddressAllocation has been successfully created/updated", + Reason: "", + LastTransitionTime: transitionTime, + }, + } + r.setReadyStatusTrue(&ctx, dummyIPAddressAllocation, transitionTime) + + if !reflect.DeepEqual(dummyIPAddressAllocation.Status.Conditions, newConditions) { + t.Fatalf("Failed to correctly update Status Conditions when conditions haven't changed") + } +} + +type fakeStatusWriter struct { +} + +func (writer fakeStatusWriter) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + return nil +} +func (writer fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return nil +} +func (writer fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + return nil +} + +type fakeRecorder struct { +} + +func (recorder fakeRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} +func (recorder fakeRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} +func (recorder fakeRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func TestIPAddressAllocationReconciler_Reconcile(t *testing.T) { + + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + + service := &ipaddressallocation.IPAddressAllocationService{ + Service: common.Service{ + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + service.NSXConfig.CoeConfig = &config.CoeConfig{} + service.NSXConfig.Cluster = "k8s_cluster" + r := &IPAddressAllocationReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + Recorder: fakeRecorder{}, + } + ctx := context.Background() + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dummy", Name: "dummy"}} + + // common.GcOnce do nothing + var once sync.Once + pat := gomonkey.ApplyMethod(reflect.TypeOf(&once), "Do", func(_ *sync.Once, _ func()) {}) + defer pat.Reset() + + // not found + errNotFound := errors.New("not found") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + _, err := r.Reconcile(ctx, req) + assert.Equal(t, err, errNotFound) + + // DeletionTimestamp.IsZero = ture, client update failed + sp := &v1alpha1.IPAddressAllocation{} + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + return nil + }) + err = errors.New("Update failed") + k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) + fakewriter := fakeStatusWriter{} + k8sClient.EXPECT().Status().Return(fakewriter) + _, ret := r.Reconcile(ctx, req) + assert.Equal(t, err, ret) + + // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.FinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + return nil + }) + + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + uid interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + + k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, uid interface{}) error { + return nil + }) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName, DeleteIPAddressAllocation fail + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + uid interface{}) error { + return errors.New("delete failed") + }) + + k8sClient.EXPECT().Status().Times(2).Return(fakewriter) + _, ret = r.Reconcile(ctx, req) + assert.NotEqual(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateIPAddressAllocation fail + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + v1sp.ObjectMeta.DeletionTimestamp = nil + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + obj *v1alpha1.IPAddressAllocation) (bool, error) { + return false, errors.New("create failed") + }) + res, ret := r.Reconcile(ctx, req) + assert.Equal(t, res, resultRequeue) + assert.NotEqual(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateIPAddressAllocation succ + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + v1sp.ObjectMeta.DeletionTimestamp = nil + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + obj *v1alpha1.IPAddressAllocation) (bool, error) { + return true, nil + }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() +} + +func TestReconciler_GarbageCollector(t *testing.T) { + // gc collect item "2345", local store has more item than k8s cache + service := &ipaddressallocation.IPAddressAllocationService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", + func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + a.Insert("1234") + a.Insert("2345") + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + return nil + }) + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + + r := &IPAddressAllocationReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + } + ctx := context.Background() + policyList := &v1alpha1.IPAddressAllocationList{} + k8sClient.EXPECT().List(gomock.Any(), policyList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + a := list.(*v1alpha1.IPAddressAllocationList) + a.Items = append(a.Items, v1alpha1.IPAddressAllocation{}) + a.Items[0].ObjectMeta = metav1.ObjectMeta{} + a.Items[0].UID = "1234" + return nil + }) + r.collectGarbage(context.Background()) + + // local store has same item as k8s cache + patch.Reset() + + patch.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + a.Insert("1234") + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + k8sClient.EXPECT().List(ctx, policyList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + a := list.(*v1alpha1.IPAddressAllocationList) + a.Items = append(a.Items, v1alpha1.IPAddressAllocation{}) + a.Items[0].ObjectMeta = metav1.ObjectMeta{} + a.Items[0].UID = "1234" + return nil + }) + r.collectGarbage(context.Background()) + + // local store has no item + patch.Reset() + + patch.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + k8sClient.EXPECT().List(ctx, policyList).Return(nil).Times(0) + r.collectGarbage(context.Background()) + + patch.Reset() +} diff --git a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go index 73c4c30a6..47170396d 100644 --- a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go +++ b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go @@ -875,7 +875,8 @@ func TestNSXServiceAccountReconciler_garbageCollector(t *testing.T) { }) }, args: args{ - nsxServiceAccountUIDSet: sets.New[string]("00000000-0000-0000-0000-000000000002", "00000000-0000-0000-0000-000000000003", "00000000-0000-0000-0000-000000000004"), + nsxServiceAccountUIDSet: sets.New[string]("00000000-0000-0000-0000-000000000002", "00000000-0000-0000-0000-000000000003", + "00000000-0000-0000-0000-000000000004"), nsxServiceAccountList: &nsxvmwarecomv1alpha1.NSXServiceAccountList{Items: []nsxvmwarecomv1alpha1.NSXServiceAccount{{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns1", diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 412b0bd1e..9bc961f36 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -71,19 +71,20 @@ type Client struct { VPCSecurityClient vpcs.SecurityPoliciesClient VPCRuleClient vpc_sp.RulesClient - OrgRootClient nsx_policy.OrgRootClient - ProjectInfraClient projects.InfraClient - VPCClient projects.VpcsClient - IPBlockClient infra.IpBlocksClient - StaticRouteClient vpcs.StaticRoutesClient - NATRuleClient nat.NatRulesClient - VpcGroupClient vpcs.GroupsClient - PortClient subnets.PortsClient - PortStateClient ports.StateClient - IPPoolClient subnets.IpPoolsClient - IPAllocationClient ip_pools.IpAllocationsClient - SubnetsClient vpcs.SubnetsClient - RealizedStateClient realized_state.RealizedEntitiesClient + OrgRootClient nsx_policy.OrgRootClient + ProjectInfraClient projects.InfraClient + VPCClient projects.VpcsClient + IPBlockClient infra.IpBlocksClient + StaticRouteClient vpcs.StaticRoutesClient + NATRuleClient nat.NatRulesClient + VpcGroupClient vpcs.GroupsClient + PortClient subnets.PortsClient + PortStateClient ports.StateClient + IPPoolClient subnets.IpPoolsClient + IPAllocationClient ip_pools.IpAllocationsClient + SubnetsClient vpcs.SubnetsClient + RealizedStateClient realized_state.RealizedEntitiesClient + IPAddressAllocationClient vpcs.IpAddressAllocationsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -205,12 +207,13 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, - NSXChecker: *nsxChecker, - NSXVerChecker: *nsxVersionChecker, - IPPoolClient: ipPoolClient, - IPAllocationClient: ipAllocationClient, - SubnetsClient: subnetsClient, - RealizedStateClient: realizedStateClient, + NSXChecker: *nsxChecker, + NSXVerChecker: *nsxVersionChecker, + IPPoolClient: ipPoolClient, + IPAllocationClient: ipAllocationClient, + SubnetsClient: subnetsClient, + RealizedStateClient: realizedStateClient, + IPAddressAllocationClient: ipAddressAllocationClient, } // NSX version check will be restarted during SecurityPolicy reconcile // So, it's unnecessary to exit even if failed in the first time diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 08d671dff..7a8ad278d 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -53,6 +53,8 @@ const ( TagScopeIPPoolCRName string = "nsx-op/ippool_name" TagScopeIPPoolCRUID string = "nsx-op/ippool_uid" TagScopeIPPoolCRType string = "nsx-op/ippool_type" + TagScopeIPAddressAllocationCRName string = "nsx-op/ipaddressallocation_name" + TagScopeIPAddressAllocationCRUID string = "nsx-op/ipaddressallocation_uid" TagScopeIPSubnetName string = "nsx-op/ipsubnet_name" TagScopeVMNamespaceUID string = "nsx-op/vm_namespace_uid" TagScopeVMNamespace string = "nsx-op/vm_namespace" @@ -84,24 +86,25 @@ const ( ValueMinorVersion string = "0" ValuePatchVersion string = "0" - GCInterval = 60 * time.Second - RealizeTimeout = 2 * time.Minute - RealizeMaxRetries = 3 - IPPoolFinalizerName = "ippool.nsx.vmware.com/finalizer" - DefaultSNATID = "DEFAULT" - AVISubnetLBID = "_AVI_SUBNET--LB" - IPPoolTypePublic = "Public" - IPPoolTypePrivate = "Private" - - SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" - NetworkPolicyFinalizerName = "networkpolicy.nsx.vmware.com/finalizer" - StaticRouteFinalizerName = "staticroute.nsx.vmware.com/finalizer" - NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" - SubnetFinalizerName = "subnet.nsx.vmware.com/finalizer" - SubnetSetFinalizerName = "subnetset.nsx.vmware.com/finalizer" - SubnetPortFinalizerName = "subnetport.nsx.vmware.com/finalizer" - NetworkInfoFinalizerName = "networkinfo.nsx.vmware.com/finalizer" - PodFinalizerName = "pod.nsx.vmware.com/finalizer" + GCInterval = 60 * time.Second + RealizeTimeout = 2 * time.Minute + RealizeMaxRetries = 3 + DefaultSNATID = "DEFAULT" + AVISubnetLBID = "_AVI_SUBNET--LB" + IPPoolTypePublic = "Public" + IPPoolTypePrivate = "Private" + + SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" + NetworkPolicyFinalizerName = "networkpolicy.nsx.vmware.com/finalizer" + StaticRouteFinalizerName = "staticroute.nsx.vmware.com/finalizer" + NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" + SubnetFinalizerName = "subnet.nsx.vmware.com/finalizer" + SubnetSetFinalizerName = "subnetset.nsx.vmware.com/finalizer" + SubnetPortFinalizerName = "subnetport.nsx.vmware.com/finalizer" + NetworkInfoFinalizerName = "networkinfo.nsx.vmware.com/finalizer" + PodFinalizerName = "pod.nsx.vmware.com/finalizer" + IPPoolFinalizerName = "ippool.nsx.vmware.com/finalizer" + IPAddressAllocationFinalizerName = "ipaddressallocation.nsx.vmware.com/finalizer" IndexKeySubnetID = "IndexKeySubnetID" IndexKeyPathPath = "Path" @@ -156,11 +159,12 @@ var ( // ResourceTypeClusterControlPlane is used by NSXServiceAccountController ResourceTypeClusterControlPlane = "clustercontrolplane" // ResourceTypePrincipalIdentity is used by NSXServiceAccountController, and it is MP resource type. - ResourceTypePrincipalIdentity = "principalidentity" - ResourceTypeSubnet = "VpcSubnet" - ResourceTypeIPPool = "IpAddressPool" - ResourceTypeIPPoolBlockSubnet = "IpAddressPoolBlockSubnet" - ResourceTypeNode = "HostTransportNode" + ResourceTypePrincipalIdentity = "principalidentity" + ResourceTypeSubnet = "VpcSubnet" + ResourceTypeIPPool = "IpAddressPool" + ResourceTypeIPAddressAllocation = "VpcIpAddressAllocation" + ResourceTypeIPPoolBlockSubnet = "IpAddressPoolBlockSubnet" + ResourceTypeNode = "HostTransportNode" ) type Service struct { diff --git a/pkg/nsx/services/ipaddressallocation/builder.go b/pkg/nsx/services/ipaddressallocation/builder.go new file mode 100644 index 000000000..742ef9e70 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/builder.go @@ -0,0 +1,49 @@ +package ipaddressallocation + +import ( + "fmt" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +var ( + Int64 = common.Int64 + String = common.String +) + +const ( + IPADDRESSALLOCATIONPREFIX = "ipa" +) + +func (service *IPAddressAllocationService) BuildIPAddressAllocation(IPAddressAllocation *v1alpha1.IPAddressAllocation) (*model.VpcIpAddressAllocation, error) { + VPCInfo := service.VPCService.ListVPCInfo(IPAddressAllocation.Namespace) + if len(VPCInfo) == 0 { + log.Error(nil, "failed to find VPCInfo for IPAddressAllocation CR", "IPAddressAllocation", IPAddressAllocation.Name, "namespace", IPAddressAllocation.Namespace) + return nil, fmt.Errorf("failed to find VPCInfo for IPAddressAllocation CR %s in namespace %s", IPAddressAllocation.Name, IPAddressAllocation.Namespace) + } + + ipAddressBlockVisibility := util.ToUpper(IPAddressAllocation.Spec.IPAddressBlockVisibility) + return &model.VpcIpAddressAllocation{ + Id: String(service.buildIPAddressAllocationID(IPAddressAllocation)), + DisplayName: String(service.buildIPAddressAllocationName(IPAddressAllocation)), + Tags: service.buildIPAddressAllocationTags(IPAddressAllocation), + IpAddressBlockVisibility: &ipAddressBlockVisibility, + AllocationSize: Int64(int64(IPAddressAllocation.Spec.AllocationSize)), + }, nil +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationID(IPAddressAllocation *v1alpha1.IPAddressAllocation) string { + return util.GenerateID(string(IPAddressAllocation.UID), IPADDRESSALLOCATIONPREFIX, "", "") +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationName(IPAddressAllocation *v1alpha1.IPAddressAllocation) string { + return util.GenerateDisplayName(IPAddressAllocation.ObjectMeta.Name, IPADDRESSALLOCATIONPREFIX, "", "", service.NSXConfig.Cluster) +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationTags(IPAddressAllocation *v1alpha1.IPAddressAllocation) []model.Tag { + return util.BuildBasicTags(service.NSXConfig.Cluster, IPAddressAllocation, "") +} diff --git a/pkg/nsx/services/ipaddressallocation/builder_test.go b/pkg/nsx/services/ipaddressallocation/builder_test.go new file mode 100644 index 000000000..99b5922a4 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/builder_test.go @@ -0,0 +1,115 @@ +package ipaddressallocation + +import ( + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/vpcclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +type fakeQueryClient struct { +} + +func (qIface *fakeQueryClient) List(_ string, _ *string, _ *string, _ *int64, _ *bool, _ *string) (model.SearchResponse, error) { + cursor := "2" + resultCount := int64(2) + return model.SearchResponse{ + Results: []*data.StructValue{{}}, + Cursor: &cursor, ResultCount: &resultCount, + }, nil +} + +func createService(t *testing.T) (*vpc.VPCService, *gomock.Controller, *mocks.MockVpcsClient) { + config2 := nsx.NewConfig("localhost", "1", "1", []string{}, 10, 3, 20, 20, true, true, true, ratelimiter.AIMD, nil, nil, []string{}) + + cluster, _ := nsx.NewCluster(config2) + rc, _ := cluster.NewRestConnector() + + mockCtrl := gomock.NewController(t) + mockVpcclient := mocks.NewMockVpcsClient(mockCtrl) + k8sClient := mock_client.NewMockClient(mockCtrl) + + vpcStore := &vpc.VPCStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeStaticRouteCRUID: indexFunc}), + BindingType: model.VpcBindingType(), + }} + + service := &vpc.VPCService{ + Service: common.Service{ + Client: k8sClient, + NSXClient: &nsx.Client{ + QueryClient: &fakeQueryClient{}, + VPCClient: mockVpcclient, + RestConnector: rc, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + NSXConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + VpcStore: vpcStore, + VPCNetworkConfigStore: vpc.VPCNetworkInfoStore{ + VPCNetworkConfigMap: map[string]common.VPCNetworkConfigInfo{}, + }, + VPCNSNetworkConfigStore: vpc.VPCNsNetworkConfigStore{ + VPCNSNetworkConfigMap: map[string]string{}, + }, + } + return service, mockCtrl, mockVpcclient +} + +func TestBuildIPAddressAllocation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vpcService, _, _ := createService(t) + ipAllocService := &IPAddressAllocationService{ + VPCService: vpcService, + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + CoeConfig: &config.CoeConfig{ + Cluster: "default", + }, + }, + }, + } + + t.Run("VPCInfo is empty", func(t *testing.T) { + ipAlloc := &v1alpha1.IPAddressAllocation{} + ipAlloc.Namespace = "default" + ipAlloc.Name = "test-ip-alloc" + + patch := gomonkey.ApplyMethod(reflect.TypeOf(ipAllocService.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, _ string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{} + }) + defer patch.Reset() + + result, err := ipAllocService.BuildIPAddressAllocation(ipAlloc) + assert.Nil(t, result) + assert.EqualError(t, err, "failed to find VPCInfo for IPAddressAllocation CR test-ip-alloc in namespace default") + }) +} diff --git a/pkg/nsx/services/ipaddressallocation/compare.go b/pkg/nsx/services/ipaddressallocation/compare.go new file mode 100644 index 000000000..d490986d8 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/compare.go @@ -0,0 +1,32 @@ +package ipaddressallocation + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +type ( + IpAddressAllocation model.VpcIpAddressAllocation +) + +type Comparable = common.Comparable + +func (iap *IpAddressAllocation) Key() string { + return *iap.Id +} + +func (iap *IpAddressAllocation) Value() data.DataValue { + s := &IpAddressAllocation{Id: iap.Id, DisplayName: iap.DisplayName, Tags: iap.Tags, AllocationSize: iap.AllocationSize, IpAddressBlockVisibility: iap.IpAddressBlockVisibility} + dataValue, _ := ComparableToIpAddressAllocation(s).GetDataValue__() + return dataValue +} + +func IpAddressAllocationToComparable(iap *model.VpcIpAddressAllocation) Comparable { + return (*IpAddressAllocation)(iap) +} + +func ComparableToIpAddressAllocation(iap Comparable) *model.VpcIpAddressAllocation { + return (*model.VpcIpAddressAllocation)(iap.(*IpAddressAllocation)) +} diff --git a/pkg/nsx/services/ipaddressallocation/compare_test.go b/pkg/nsx/services/ipaddressallocation/compare_test.go new file mode 100644 index 000000000..7e18ba86b --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/compare_test.go @@ -0,0 +1,52 @@ +package ipaddressallocation + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func TestKey(t *testing.T) { + id := "test-id" + iap := &IpAddressAllocation{Id: &id} + assert.Equal(t, "test-id", iap.Key()) + + iapNil := &IpAddressAllocation{Id: nil} + assert.Panics(t, func() { iapNil.Key() }) +} + +func TestValue(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + iap := &IpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + dataValue := iap.Value() + expectedDataValue, _ := ComparableToIpAddressAllocation(iap).GetDataValue__() + + assert.Equal(t, expectedDataValue, dataValue) +} + +func TestIpAddressAllocationToComparable(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + vpcIap := &model.VpcIpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + comparable := IpAddressAllocationToComparable(vpcIap) + assert.IsType(t, &IpAddressAllocation{}, comparable) +} + +func TestComparableToIpAddressAllocation(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + iap := &IpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + vpcIap := ComparableToIpAddressAllocation(iap) + assert.IsType(t, &model.VpcIpAddressAllocation{}, vpcIap) + assert.Equal(t, id, *vpcIap.Id) + assert.Equal(t, displayName, *vpcIap.DisplayName) + assert.Equal(t, tags, vpcIap.Tags) +} diff --git a/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go new file mode 100644 index 000000000..f013ec2a0 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go @@ -0,0 +1,199 @@ +package ipaddressallocation + +import ( + "context" + "fmt" + "sync" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +var ( + log = logger.Log + MarkedForDelete = true + ResourceTypeIPAddressAllocation = common.ResourceTypeIPAddressAllocation +) + +type IPAddressAllocationService struct { + common.Service + ipAddressAllocationStore *IPAddressAllocationStore + VPCService common.VPCServiceProvider +} + +func InitializeIPAddressAllocation(service common.Service, vpcService common.VPCServiceProvider) (*IPAddressAllocationService, error) { + wg := sync.WaitGroup{} + wgDone := make(chan bool) + fatalErrors := make(chan error) + + wg.Add(1) + + ipAddressAllocationService := &IPAddressAllocationService{Service: service, VPCService: vpcService} + ipAddressAllocationService.ipAddressAllocationStore = &IPAddressAllocationStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}), + BindingType: model.VpcIpAddressAllocationBindingType(), + }} + + tags := []model.Tag{ + {Scope: String(common.TagScopeIPAddressAllocationCRUID)}, + } + go ipAddressAllocationService.InitializeResourceStore(&wg, fatalErrors, ResourceTypeIPAddressAllocation, tags, ipAddressAllocationService.ipAddressAllocationStore) + + go func() { + wg.Wait() + close(wgDone) + }() + select { + case <-wgDone: + break + case err := <-fatalErrors: + close(fatalErrors) + return ipAddressAllocationService, err + } + return ipAddressAllocationService, nil +} + +func (service *IPAddressAllocationService) CreateOrUpdateIPAddressAllocation(obj *v1alpha1.IPAddressAllocation) (bool, error) { + nsxIPAddressAllocation, err := service.BuildIPAddressAllocation(obj) + if err != nil { + return false, err + } + existingIPAddressAllocation, err := service.indexedIPAddressAllocation(obj.UID) + if err != nil { + log.Error(err, "failed to get ipaddressallocation", "UID", obj.UID) + return false, err + } + log.V(1).Info("existing ipaddressallocation", "ipaddressallocation", existingIPAddressAllocation) + ipAddressAllocationUpdated := common.CompareResource(IpAddressAllocationToComparable(existingIPAddressAllocation), + IpAddressAllocationToComparable(nsxIPAddressAllocation)) + + if !ipAddressAllocationUpdated { + log.Info("ipaddressallocation is not changed", "UID", obj.UID) + return false, nil + } + + if err := service.Apply(nsxIPAddressAllocation); err != nil { + return false, err + } + + createdIPAddressAllocation, err := service.indexedIPAddressAllocation(obj.UID) + if err != nil { + log.Error(err, "failed to get created ipaddressallocation", "UID", obj.UID) + return false, err + } + cidr := createdIPAddressAllocation.AllocationIps + if cidr == nil { + return false, fmt.Errorf("ipaddressallocation %s didn't realize available cidr", obj.UID) + } + obj.Status.CIDR = *cidr + return true, nil +} + +func (service *IPAddressAllocationService) Apply(nsxIPAddressAllocation *model.VpcIpAddressAllocation) error { + ns := service.GetIPAddressAllocationNamespace(nsxIPAddressAllocation) + var err error + VPCInfo := service.VPCService.ListVPCInfo(ns) + if len(VPCInfo) == 0 { + err = util.NoEffectiveOption{Desc: "no valid org and project for ipaddressallocation"} + return err + } + err = service.NSXClient.IPAddressAllocationClient.Patch(VPCInfo[0].OrgID, VPCInfo[0].ProjectID, VPCInfo[0].ID, *nsxIPAddressAllocation.Id, *nsxIPAddressAllocation) + err = util.NSXApiError(err) + if err != nil { + // not return err, try to get it from nsx, in case if cidr not realized at the first time + // so it can be patched in the next time and reacquire cidr + log.Error(err, "patch failed, try to get it from nsx", "nsxIPAddressAllocation", nsxIPAddressAllocation) + } + // get back from nsx, it contains path which is used to parse vpc info when deleting + nsxIPAddressAllocationNew, err := service.NSXClient.IPAddressAllocationClient.Get(VPCInfo[0].OrgID, VPCInfo[0].ProjectID, VPCInfo[0].ID, *nsxIPAddressAllocation.Id) + err = util.NSXApiError(err) + if err != nil { + return err + } + if nsxIPAddressAllocation.AllocationIps == nil { + err := fmt.Errorf("cidr not realized yet") + return err + } + err = service.ipAddressAllocationStore.Apply(&nsxIPAddressAllocationNew) + if err != nil { + return err + } + log.V(1).Info("successfully created or updated ipaddressallocation", "nsxIPAddressAllocation", nsxIPAddressAllocation) + return nil +} + +func (service *IPAddressAllocationService) DeleteIPAddressAllocation(obj interface{}) error { + var err error + var nsxIPAddressAllocation *model.VpcIpAddressAllocation + switch o := obj.(type) { + case *v1alpha1.IPAddressAllocation: + nsxIPAddressAllocation, err = service.indexedIPAddressAllocation(o.UID) + if err != nil { + log.Error(err, "failed to get ipaddressallocation", "IPAddressAllocation", o) + return err + } + case types.UID: + nsxIPAddressAllocation, err = service.indexedIPAddressAllocation(o) + if err != nil { + log.Error(err, "failed to get ipaddressallocation by UID", "UID", o) + return err + } + } + if nsxIPAddressAllocation == nil { + log.Error(nil, "failed to get ipaddressallocation from store, skip") + return nil + } + vpcResourceInfo, err := common.ParseVPCResourcePath(*nsxIPAddressAllocation.Path) + if err != nil { + return err + } + err = service.NSXClient.IPAddressAllocationClient.Delete(vpcResourceInfo.OrgID, vpcResourceInfo.ProjectID, vpcResourceInfo.ID, *nsxIPAddressAllocation.Id) + if err != nil { + return err + } + nsxIPAddressAllocation.MarkedForDelete = &MarkedForDelete + err = service.ipAddressAllocationStore.Apply(nsxIPAddressAllocation) + if err != nil { + return err + } + log.V(1).Info("successfully deleted nsxIPAddressAllocation", "nsxIPAddressAllocation", nsxIPAddressAllocation) + return nil +} + +func (service *IPAddressAllocationService) ListIPAddressAllocationID() sets.Set[string] { + ipAddressAllocationSet := service.ipAddressAllocationStore.ListIndexFuncValues(common.TagScopeIPAddressAllocationCRUID) + return ipAddressAllocationSet +} + +func (service *IPAddressAllocationService) GetIPAddressAllocationNamespace(nsxIPAddressAllocation *model.VpcIpAddressAllocation) string { + for _, tag := range nsxIPAddressAllocation.Tags { + if *tag.Scope == common.TagScopeNamespace { + return *tag.Tag + } + } + return "" +} + +func (service *IPAddressAllocationService) Cleanup(ctx context.Context) error { + uids := service.ListIPAddressAllocationID() + log.Info("cleaning up ipaddressallocation", "count", len(uids)) + for uid := range uids { + select { + case <-ctx.Done(): + return util.TimeoutFailed + default: + err := service.DeleteIPAddressAllocation(types.UID(uid)) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/nsx/services/ipaddressallocation/store.go b/pkg/nsx/services/ipaddressallocation/store.go new file mode 100644 index 000000000..5b0047374 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/store.go @@ -0,0 +1,90 @@ +package ipaddressallocation + +import ( + "errors" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func keyFunc(obj interface{}) (string, error) { + switch v := obj.(type) { + case *model.VpcIpAddressAllocation: + return *v.Id, nil + case *model.GenericPolicyRealizedResource: + return *v.Id, nil + default: + return "", errors.New("keyFunc doesn't support unknown type") + } +} + +func indexFunc(obj interface{}) ([]string, error) { + res := make([]string, 0, 5) + switch v := obj.(type) { + case *model.VpcIpAddressAllocation: + return filterTag(v.Tags), nil + case *model.GenericPolicyRealizedResource: + return filterTag(v.Tags), nil + default: + return res, errors.New("indexFunc doesn't support unknown type") + } +} + +var filterTag = func(v []model.Tag) []string { + res := make([]string, 0, 5) + for _, tag := range v { + if *tag.Scope == common.TagScopeIPAddressAllocationCRUID { + res = append(res, *tag.Tag) + } + } + return res +} + +type IPAddressAllocationStore struct { + common.ResourceStore +} + +func (ipAddressAllocationStore *IPAddressAllocationStore) Apply(i interface{}) error { + ipAddressAllocation := i.(*model.VpcIpAddressAllocation) + if ipAddressAllocation.MarkedForDelete != nil && *ipAddressAllocation.MarkedForDelete { + err := ipAddressAllocationStore.Delete(ipAddressAllocation) + if err != nil { + return err + } + log.V(1).Info("delete ipAddressAllocation from store", "ipAddressAllocation", ipAddressAllocation) + } else { + err := ipAddressAllocationStore.Add(ipAddressAllocation) + if err != nil { + return err + } + log.V(1).Info("add ipAddressAllocation to store", "ipAddressAllocation", ipAddressAllocation) + } + return nil +} + +func (service *IPAddressAllocationService) indexedIPAddressAllocation(uid types.UID) (*model.VpcIpAddressAllocation, error) { + nsxIPAddressAllocation, err := service.ipAddressAllocationStore.GetByIndex(uid) + if err != nil { + return nil, err + } + return nsxIPAddressAllocation, nil +} + +func (ipAddressAllocationStore *IPAddressAllocationStore) GetByIndex(uid types.UID) (*model.VpcIpAddressAllocation, error) { + nsxIPAddressAllocation := &model.VpcIpAddressAllocation{} + indexResults, err := ipAddressAllocationStore.ResourceStore.ByIndex(common.TagScopeIPAddressAllocationCRUID, string(uid)) + if err != nil { + log.Error(err, "failed to get ipaddressallocation", "UID", string(uid)) + return nil, err + } + if len(indexResults) > 0 { + t := indexResults[0].(*model.VpcIpAddressAllocation) + nsxIPAddressAllocation = t + } else { + log.Info("did not get ipaddressallocation with index", "UID", string(uid)) + return nsxIPAddressAllocation, nil + } + return nsxIPAddressAllocation, nil +} diff --git a/pkg/nsx/services/ipaddressallocation/store_test.go b/pkg/nsx/services/ipaddressallocation/store_test.go new file mode 100644 index 000000000..002f6a429 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/store_test.go @@ -0,0 +1,98 @@ +package ipaddressallocation + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func TestIPAddressAllocationStore_CRUDResource(t *testing.T) { + ipAddressAllocationCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}) + resourceStore := common.ResourceStore{ + Indexer: ipAddressAllocationCacheIndexer, + BindingType: model.VpcIpAddressAllocationBindingType(), + } + ipAddressAllocationStore := &IPAddressAllocationStore{ResourceStore: resourceStore} + type args struct { + i interface{} + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + {"1", args{i: &model.VpcIpAddressAllocation{Id: String("1")}}, assert.NoError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantErr(t, ipAddressAllocationStore.Apply(tt.args.i), fmt.Sprintf("Apply(%v)", tt.args.i)) + }) + } +} + +func TestIPAddressAllocationStore_GetByIndex(t *testing.T) { + p := &model.VpcIpAddressAllocation{Id: String("1"), DisplayName: String("1"), + Tags: []model.Tag{{Scope: String(common.TagScopeIPAddressAllocationCRUID), + Tag: String("1")}}} + ipAddressAllocationStore := &IPAddressAllocationStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}), + BindingType: model.VpcIpAddressBindingType(), + }} + _ = ipAddressAllocationStore.Apply(p) + type args struct { + uid types.UID + } + tests := []struct { + name string + args args + want *model.VpcIpAddressAllocation + wantErr bool + }{ + {"1", args{uid: "1"}, &model.VpcIpAddressAllocation{Id: String("1"), DisplayName: String("1"), + Tags: []model.Tag{{Scope: String(common.TagScopeIPAddressAllocationCRUID), Tag: String("1")}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ipAddressAllocationStore.GetByIndex(tt.args.uid) + if (err != nil) != tt.wantErr { + t.Errorf("indexedIPAddressAllocation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("indexedIPAddressAllocation() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_indexFunc(t *testing.T) { + mId, mTag, mScope := "11111", "11111", common.TagScopeIPAddressAllocationCRUID + m := &model.VpcIpAddressAllocation{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + } + t.Run("1", func(t *testing.T) { + got, _ := indexFunc(m) + if !reflect.DeepEqual(got, []string{"11111"}) { + t.Errorf("indexFunc() = %v, want %v", got, model.Tag{Tag: &mTag, Scope: &mScope}) + } + }) +} + +func Test_keyFunc(t *testing.T) { + Id := "11111" + g := &model.VpcIpAddressAllocation{Id: &Id} + t.Run("2", func(t *testing.T) { + got, _ := keyFunc(g) + if got != "11111" { + t.Errorf("keyFunc() = %v, want %v", got, "11111") + } + }) +} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 0b537b313..e6060c6fa 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -477,6 +477,10 @@ func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []mo tags = append(tags, model.Tag{Scope: String(common.TagScopeNamespace), Tag: String(i.ObjectMeta.Namespace)}) tags = append(tags, model.Tag{Scope: String(common.TagScopeIPPoolCRName), Tag: String(i.ObjectMeta.Name)}) tags = append(tags, model.Tag{Scope: String(common.TagScopeIPPoolCRUID), Tag: String(string(i.UID))}) + case *v1alpha1.IPAddressAllocation: + tags = append(tags, model.Tag{Scope: String(common.TagScopeNamespace), Tag: String(i.ObjectMeta.Namespace)}) + tags = append(tags, model.Tag{Scope: String(common.TagScopeIPAddressAllocationCRName), Tag: String(i.ObjectMeta.Name)}) + tags = append(tags, model.Tag{Scope: String(common.TagScopeIPAddressAllocationCRUID), Tag: String(string(i.UID))}) default: log.Info("unknown obj type", "obj", obj) }