From 752934f5bdfad56c896b37edc015c0ae4cd947f1 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Thu, 16 May 2024 11:09:22 +0800 Subject: [PATCH] Implement ipaddressallocation controller Signed-off-by: Xie Zheng --- .github/workflows/makefile.yml | 2 +- .../nsx.vmware.com_ipaddressallocations.yaml | 32 +- cmd/main.go | 22 ++ .../v1alpha1/ipaddressallocation_types.go | 4 +- .../v1alpha1/zz_generated.deepcopy.go | 185 +++++----- .../v1alpha1/ipaddressallocation_types.go | 9 +- pkg/apis/v1alpha1/zz_generated.deepcopy.go | 185 +++++----- .../ipaddressallocation_controller.go | 245 +++++++++++++ .../ipaddressallocation_controller_test.go | 345 ++++++++++++++++++ .../nsxserviceaccount_controller.go | 3 +- .../nsxserviceaccount_controller_test.go | 3 +- pkg/nsx/client.go | 41 ++- pkg/nsx/services/common/types.go | 31 +- .../services/ipaddressallocation/builder.go | 74 ++++ .../ipaddressallocation/builder_test.go | 163 +++++++++ .../services/ipaddressallocation/compare.go | 32 ++ .../ipaddressallocation/compare_test.go | 52 +++ .../ipaddressallocation.go | 197 ++++++++++ pkg/nsx/services/ipaddressallocation/store.go | 89 +++++ .../ipaddressallocation/store_test.go | 98 +++++ pkg/util/utils.go | 4 + 21 files changed, 1577 insertions(+), 239 deletions(-) create mode 100644 pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go create mode 100644 pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/builder.go create mode 100644 pkg/nsx/services/ipaddressallocation/builder_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/compare.go create mode 100644 pkg/nsx/services/ipaddressallocation/compare_test.go create mode 100644 pkg/nsx/services/ipaddressallocation/ipaddressallocation.go create mode 100644 pkg/nsx/services/ipaddressallocation/store.go create mode 100644 pkg/nsx/services/ipaddressallocation/store_test.go diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index eb4851cc5..a8e6eeec1 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.21.9 - name: Run golangci-lint run: make golangci diff --git a/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml b/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml index 476fa42d6..485d9707f 100644 --- a/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml +++ b/build/yaml/crd/nsx.vmware.com_ipaddressallocations.yaml @@ -3,7 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null name: ipaddressallocations.nsx.vmware.com spec: group: nsx.vmware.com @@ -29,19 +30,14 @@ spec: description: IPAddressAllocation is the Schema for the IP allocation API. properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -54,12 +50,10 @@ spec: ip_address_block_visibility: default: Private description: IPAddressBlockVisibility specifies the visibility of - the IPBlocks to allocate IP addresses. Can be External, Private - or Project. + the IPBlocks to allocate IP addresses. Can be External or Private. enum: - External - Private - - Project type: string type: object status: @@ -73,10 +67,10 @@ spec: description: Condition defines condition of custom resource. properties: lastTransitionTime: - description: |- - Last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when - the API field changed is acceptable. + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. format: date-time type: string message: diff --git a/cmd/main.go b/cmd/main.go index 1abec4603..a6478f1b4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2" + "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ipaddressallocation" "github.com/vmware-tanzu/nsx-operator/pkg/config" ippool2 "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ippool" @@ -39,6 +40,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/metrics" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + ipaddressallocationservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ippool" nodeservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/node" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/nsxserviceaccount" @@ -147,6 +149,21 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp } } +func StartIPAddressAllocationController(mgr ctrl.Manager, ipAddressAllocationService *ipaddressallocationservice.IPAddressAllocationService, vpcService common.VPCServiceProvider) { + ipAddressAllocationReconciler := &ipaddressallocation.IPAddressAllocationReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Service: ipAddressAllocationService, + VPCService: vpcService, + Recorder: mgr.GetEventRecorderFor("ipaddressallocation-controller"), + } + + if err := ipAddressAllocationReconciler.SetupWithManager(mgr); err != nil { + log.Error(err, "failed to create namespace controller", "controller", "Namespace") + os.Exit(1) + } +} + func main() { log.Info("starting NSX Operator") mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -202,6 +219,10 @@ func main() { if err != nil { log.Error(err, "failed to initialize ippool commonService", "controller", "IPPool") } + ipAddressAllocationService, err := ipaddressallocationservice.InitializeIPAddressAllocation(commonService, vpcService) + if err != nil { + log.Error(err, "failed to initialize ipaddressallocation commonService", "controller", "IPAddressAllocation") + } subnetPortService, err := subnetportservice.InitializeSubnetPort(commonService) if err != nil { log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort") @@ -238,6 +259,7 @@ func main() { subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService) pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService) StartIPPoolController(mgr, ipPoolService, vpcService) + StartIPAddressAllocationController(mgr, ipAddressAllocationService, vpcService) networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService) service.StartServiceLbController(mgr, commonService) } diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go b/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go index 6a50925c5..5e6ff5905 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/ipaddressallocation_types.go @@ -42,8 +42,8 @@ type IPAddressAllocationList struct { // IPAddressAllocationSpec defines the desired state of IPAddressAllocation. type IPAddressAllocationSpec struct { - // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project. - // +kubebuilder:validation:Enum=External;Private;Project + // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private. + // +kubebuilder:validation:Enum=External;Private // +kubebuilder:default=Private // +optional IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"` diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/nsx.vmware.com/v1alpha1/zz_generated.deepcopy.go index 5919f2d28..0fe16f6bd 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/zz_generated.deepcopy.go @@ -80,6 +80,102 @@ func (in *DNSClientConfig) DeepCopy() *DNSClientConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. +func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { + if in == nil { + return nil + } + out := new(IPAddressAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPAddressAllocation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. +func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { + if in == nil { + return nil + } + out := new(IPAddressAllocationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. +func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { + if in == nil { + return nil + } + out := new(IPAddressAllocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. +func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { + if in == nil { + return nil + } + out := new(IPAddressAllocationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IPBlock) DeepCopyInto(out *IPBlock) { *out = *in @@ -1395,92 +1491,3 @@ func (in *VPCState) DeepCopy() *VPCState { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. -func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { - if in == nil { - return nil - } - out := new(IPAddressAllocation) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]IPAddressAllocation, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. -func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { - if in == nil { - return nil - } - out := new(IPAddressAllocationList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. -func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { - if in == nil { - return nil - } - out := new(IPAddressAllocationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. -func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { - if in == nil { - return nil - } - out := new(IPAddressAllocationStatus) - in.DeepCopyInto(out) - return out -} \ No newline at end of file diff --git a/pkg/apis/v1alpha1/ipaddressallocation_types.go b/pkg/apis/v1alpha1/ipaddressallocation_types.go index 6a50925c5..4c50d0e2d 100644 --- a/pkg/apis/v1alpha1/ipaddressallocation_types.go +++ b/pkg/apis/v1alpha1/ipaddressallocation_types.go @@ -10,9 +10,8 @@ import ( type IPAddressVisibility string const ( - IPAddressVisibilityExternal = "External" - IPAddressVisibilityPrivate = "Private" - IPAddressVisibilityProject = "Project" + IPAddressVisibilityExternal = "EXTERNAL" + IPAddressVisibilityPrivate = "PRIVATE" ) // +genclient @@ -42,8 +41,8 @@ type IPAddressAllocationList struct { // IPAddressAllocationSpec defines the desired state of IPAddressAllocation. type IPAddressAllocationSpec struct { - // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or Project. - // +kubebuilder:validation:Enum=External;Private;Project + // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External or Private. + // +kubebuilder:validation:Enum=External;Private // +kubebuilder:default=Private // +optional IPAddressBlockVisibility IPAddressVisibility `json:"ip_address_block_visibility,omitempty"` diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 5919f2d28..0fe16f6bd 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -80,6 +80,102 @@ func (in *DNSClientConfig) DeepCopy() *DNSClientConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. +func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { + if in == nil { + return nil + } + out := new(IPAddressAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPAddressAllocation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. +func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { + if in == nil { + return nil + } + out := new(IPAddressAllocationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. +func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { + if in == nil { + return nil + } + out := new(IPAddressAllocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. +func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { + if in == nil { + return nil + } + out := new(IPAddressAllocationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IPBlock) DeepCopyInto(out *IPBlock) { *out = *in @@ -1395,92 +1491,3 @@ func (in *VPCState) DeepCopy() *VPCState { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. -func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { - if in == nil { - return nil - } - out := new(IPAddressAllocation) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]IPAddressAllocation, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. -func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { - if in == nil { - return nil - } - out := new(IPAddressAllocationList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. -func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { - if in == nil { - return nil - } - out := new(IPAddressAllocationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. -func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { - if in == nil { - return nil - } - out := new(IPAddressAllocationStatus) - in.DeepCopyInto(out) - return out -} \ No newline at end of file diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go new file mode 100644 index 000000000..555a7a98b --- /dev/null +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go @@ -0,0 +1,245 @@ +/* Copyright © 2023 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package ipaddressallocation + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "github.com/vmware-tanzu/nsx-operator/pkg/metrics" + servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" + util2 "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +var ( + log = logger.Log + once sync.Once + resultNormal = common.ResultNormal + resultRequeue = common.ResultRequeue + MetricResType = common.MetricResTypeIPPool +) + +// IPAddressAllocationReconciler reconciles a IPAddressAllocation object +type IPAddressAllocationReconciler struct { + client.Client + Scheme *apimachineryruntime.Scheme + Service *ipaddressallocation.IPAddressAllocationService + VPCService servicecommon.VPCServiceProvider + Recorder record.EventRecorder +} + +func deleteSuccess(r *IPAddressAllocationReconciler, _ *context.Context, o *v1alpha1.IPAddressAllocation) { + r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulDelete, "IPPool CR has been successfully deleted") + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) +} + +func deleteFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) { + r.setReadyStatusFalse(c, o, metav1.Now(), e) + r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailDelete, fmt.Sprintf("%v", *e)) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) +} + +func updateSuccess(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation) { + r.setReadyStatusTrue(c, o, metav1.Now()) + r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "IPAddressAllocation CR has been successfully updated") + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResType) +} + +func updateFail(r *IPAddressAllocationReconciler, c *context.Context, o *v1alpha1.IPAddressAllocation, e *error) { + r.setReadyStatusFalse(c, o, metav1.Now(), e) + r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailUpdate, fmt.Sprintf("%v", *e)) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResType) +} + +func (r *IPAddressAllocationReconciler) setReadyStatusFalse(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time, err *error) { + conditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: "NSX IPAddressAllocation could not be created/updated/deleted", + Reason: fmt.Sprintf( + "error occurred while processing the IPAddressAllocation CR. Error: %v", + *err, + ), + LastTransitionTime: transitionTime, + }, + } + ipaddressallocation.Status.Conditions = conditions + e := r.Client.Status().Update(*ctx, ipaddressallocation) + if e != nil { + log.Error(e, "unable to update IPAddressAllocation status", "ipaddressallocation", ipaddressallocation) + } +} + +func (r *IPAddressAllocationReconciler) setReadyStatusTrue(ctx *context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time) { + conditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionTrue, + Message: "NSX IPAddressAllocation has been successfully created/updated", + Reason: "", + LastTransitionTime: transitionTime, + }, + } + ipaddressallocation.Status.Conditions = conditions + e := r.Client.Status().Update(*ctx, ipaddressallocation) + if e != nil { + log.Error(e, "unable to update IPAddressAllocation status", "ipaddressallocation", ipaddressallocation) + } +} + +func (r *IPAddressAllocationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + once.Do(func() { go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, r.collectGarbage) }) + obj := &v1alpha1.IPAddressAllocation{} + log.Info("reconciling ipaddressallocation CR", "ipaddressallocation", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) + if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { + log.Error(err, "unable to fetch ipaddressallocation CR", "req", req.NamespacedName) + return resultNormal, client.IgnoreNotFound(err) + } + if obj.ObjectMeta.DeletionTimestamp.IsZero() { + return r.handleUpdate(ctx, req, obj) + } + return r.handleDeletion(ctx, req, obj) +} + +func (r *IPAddressAllocationReconciler) handleExhaustedIPBlock(err error) error { + if errors.As(err, &util.IPBlockAllExhaustedError{}) { + r.Service.ExhaustedIPBlock = []string{} + log.Info("Clear ExhaustedIPBlock: ", "ExhaustedIPBlock", r.Service.ExhaustedIPBlock) + return err + } + + if strings.Contains(err.Error(), "610755") { + pathPattern := `IpAddressBlock \[([^\]]+)\]` + pathRegex := regexp.MustCompile(pathPattern) + pathMatch := pathRegex.FindStringSubmatch(err.Error()) + if len(pathMatch) > 1 { + path := pathMatch[1] + if !util2.Contains(r.Service.ExhaustedIPBlock, path) { + r.Service.ExhaustedIPBlock = append(r.Service.ExhaustedIPBlock, path) + log.Info("ExhaustedIPBlock: ", "ExhaustedIPBlock", r.Service.ExhaustedIPBlock) + } + } + } + return nil +} + +func (r *IPAddressAllocationReconciler) handleUpdate(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) + if !controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) { + controllerutil.AddFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) + if err := r.Client.Update(ctx, obj); err != nil { + log.Error(err, "add finalizer", "ipaddressallocation", req.NamespacedName) + updateFail(r, &ctx, obj, &err) + return resultRequeue, err + } + log.V(1).Info("added finalizer on ipaddressallocation CR", "ipaddressallocation", req.NamespacedName) + } + + if err := r.Service.CreateOrUpdateIPAddressAllocation(obj); err != nil { + updateFail(r, &ctx, obj, &err) + err = r.handleExhaustedIPBlock(err) + if err != nil { + return common.ResultRequeueAfter10sec, err + } + return resultRequeue, nil + } + updateSuccess(r, &ctx, obj) + return resultNormal, nil +} + +func (r *IPAddressAllocationReconciler) handleDeletion(ctx context.Context, req ctrl.Request, obj *v1alpha1.IPAddressAllocation) (ctrl.Result, error) { + if controllerutil.ContainsFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + if err := r.Service.DeleteIPAddressAllocation(obj); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "ipaddressallocation", req.NamespacedName) + deleteFail(r, &ctx, obj, &err) + return resultRequeue, err + } + controllerutil.RemoveFinalizer(obj, servicecommon.IPAddressAllocationFinalizerName) + if err := r.Client.Update(ctx, obj); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "ipaddressallocation", req.NamespacedName) + deleteFail(r, &ctx, obj, &err) + return resultRequeue, err + } + log.V(1).Info("removed finalizer on ipaddressallocation CR", "ipaddressallocation", req.NamespacedName) + deleteSuccess(r, &ctx, obj) + log.Info("successfully deleted ipaddressallocation CR and all subnets", "ipaddressallocation", obj) + } else { + // only print a message because it's not a normal case + log.Info("ipaddressallocation CR is being deleted but its finalizers cannot be recognized", "ipaddressallocation", req.NamespacedName) + } + return resultNormal, nil +} + +func (r *IPAddressAllocationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.IPAddressAllocation{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Ignore updates to CR status in which case metadata.Generation does not change + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Suppress Delete events to avoid filtering them out in the Reconcile function + return false + }, + }). + WithOptions( + controller.Options{ + MaxConcurrentReconciles: common.NumReconcile(), + }). + Complete(r) +} + +func (r *IPAddressAllocationReconciler) collectGarbage(ctx context.Context) { + log.Info("ipaddressallocation garbage collector started") + ipAddressAllocationSet := r.Service.ListIPAddressAllocationID() + if len(ipAddressAllocationSet) == 0 { + return + } + + ipAddressAllocationList := &v1alpha1.IPAddressAllocationList{} + if err := r.Client.List(ctx, ipAddressAllocationList); err != nil { + log.Error(err, "failed to list ipaddressallocation CR") + return + } + CRIPAddressAllocationSet := sets.New[string]() + for _, ipa := range ipAddressAllocationList.Items { + CRIPAddressAllocationSet.Insert(string(ipa.UID)) + } + + log.V(2).Info("ipaddressallocation garbage collector", "nsxIPAddressAllocationSet", ipAddressAllocationSet, "CRIPAddressAllocationSet", CRIPAddressAllocationSet) + + diffSet := ipAddressAllocationSet.Difference(CRIPAddressAllocationSet) + for elem := range diffSet { + log.Info("GC collected ipaddressallocation CR", "UID", elem) + if err := r.Service.DeleteIPAddressAllocation(types.UID(elem)); err != nil { + log.Error(err, "failed to delete ipaddressallocation CR", "UID", elem) + } + } +} diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go new file mode 100644 index 000000000..a27c6c9fb --- /dev/null +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go @@ -0,0 +1,345 @@ +/* Copyright © 2021 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package ipaddressallocation + +import ( + "context" + "errors" + "reflect" + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + _ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipaddressallocation" +) + +func NewFakeIPAddressAllocationReconciler() *IPAddressAllocationReconciler { + return &IPAddressAllocationReconciler{ + Client: fake.NewClientBuilder().Build(), + Scheme: fake.NewClientBuilder().Build().Scheme(), + Service: nil, + } +} + +func TestIPAddressAllocationController_setReadyStatusTrue(t *testing.T) { + r := NewFakeIPAddressAllocationReconciler() + ctx := context.TODO() + dummyIPAddressAllocation := &v1alpha1.IPAddressAllocation{} + transitionTime := metav1.Now() + + // Case: Static Route CRD creation fails + newConditions := []v1alpha1.Condition{ + { + Type: v1alpha1.Ready, + Status: v1.ConditionTrue, + Message: "NSX IPAddressAllocation has been successfully created/updated", + Reason: "", + LastTransitionTime: transitionTime, + }, + } + r.setReadyStatusTrue(&ctx, dummyIPAddressAllocation, transitionTime) + + if !reflect.DeepEqual(dummyIPAddressAllocation.Status.Conditions, newConditions) { + t.Fatalf("Failed to correctly update Status Conditions when conditions haven't changed") + } +} + +type fakeStatusWriter struct { +} + +func (writer fakeStatusWriter) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + return nil +} +func (writer fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return nil +} +func (writer fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + return nil +} + +type fakeRecorder struct { +} + +func (recorder fakeRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} +func (recorder fakeRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} +func (recorder fakeRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func TestIPAddressAllocationReconciler_Reconcile(t *testing.T) { + + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + + service := &ipaddressallocation.IPAddressAllocationService{ + Service: common.Service{ + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + service.NSXConfig.CoeConfig = &config.CoeConfig{} + service.NSXConfig.Cluster = "k8s_cluster" + r := &IPAddressAllocationReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + Recorder: fakeRecorder{}, + } + ctx := context.Background() + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dummy", Name: "dummy"}} + + // common.GcOnce do nothing + var once sync.Once + pat := gomonkey.ApplyMethod(reflect.TypeOf(&once), "Do", func(_ *sync.Once, _ func()) {}) + defer pat.Reset() + + // not found + errNotFound := errors.New("not found") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + _, err := r.Reconcile(ctx, req) + assert.Equal(t, err, errNotFound) + + // DeletionTimestamp.IsZero = ture, client update failed + sp := &v1alpha1.IPAddressAllocation{} + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + return nil + }) + err = errors.New("Update failed") + k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) + fakewriter := fakeStatusWriter{} + k8sClient.EXPECT().Status().Return(fakewriter) + _, ret := r.Reconcile(ctx, req) + assert.Equal(t, err, ret) + + // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.FinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + return nil + }) + + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + uid interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + + k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, uid interface{}) error { + return nil + }) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName, DeleteIPAddressAllocation fail + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + uid interface{}) error { + return errors.New("delete failed") + }) + + k8sClient.EXPECT().Status().Times(2).Return(fakewriter) + _, ret = r.Reconcile(ctx, req) + assert.NotEqual(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateIPAddressAllocation fail + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + v1sp.ObjectMeta.DeletionTimestamp = nil + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + obj *v1alpha1.IPAddressAllocation) error { + return errors.New("create failed") + }) + res, ret := r.Reconcile(ctx, req) + assert.Equal(t, res, resultRequeue) + assert.Equal(t, ret, nil) + patch.Reset() + + // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateIPAddressAllocation succ + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.IPAddressAllocation) + v1sp.ObjectMeta.DeletionTimestamp = nil + v1sp.Finalizers = []string{common.IPAddressAllocationFinalizerName} + return nil + }) + + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, + obj *v1alpha1.IPAddressAllocation) error { + return nil + }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, nil) + patch.Reset() +} + +func TestReconciler_GarbageCollector(t *testing.T) { + // gc collect item "2345", local store has more item than k8s cache + service := &ipaddressallocation.IPAddressAllocationService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", + func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + a.Insert("1234") + a.Insert("2345") + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + return nil + }) + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + + r := &IPAddressAllocationReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + } + ctx := context.Background() + policyList := &v1alpha1.IPAddressAllocationList{} + k8sClient.EXPECT().List(gomock.Any(), policyList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + a := list.(*v1alpha1.IPAddressAllocationList) + a.Items = append(a.Items, v1alpha1.IPAddressAllocation{}) + a.Items[0].ObjectMeta = metav1.ObjectMeta{} + a.Items[0].UID = "1234" + return nil + }) + r.collectGarbage(context.Background()) + + // local store has same item as k8s cache + patch.Reset() + + patch.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + a.Insert("1234") + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + k8sClient.EXPECT().List(ctx, policyList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + a := list.(*v1alpha1.IPAddressAllocationList) + a.Items = append(a.Items, v1alpha1.IPAddressAllocation{}) + a.Items[0].ObjectMeta = metav1.ObjectMeta{} + a.Items[0].UID = "1234" + return nil + }) + r.collectGarbage(context.Background()) + + // local store has no item + patch.Reset() + + patch.ApplyMethod(reflect.TypeOf(service), "ListIPAddressAllocationID", func(_ *ipaddressallocation.IPAddressAllocationService) sets.Set[string] { + a := sets.New[string]() + return a + }) + patch.ApplyMethod(reflect.TypeOf(service), "DeleteIPAddressAllocation", func(_ *ipaddressallocation.IPAddressAllocationService, UID interface{}) error { + assert.FailNow(t, "should not be called") + return nil + }) + k8sClient.EXPECT().List(ctx, policyList).Return(nil).Times(0) + r.collectGarbage(context.Background()) + + patch.Reset() +} + +func TestSetupWithManager(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + // Create a mock manager + mgr, _ := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + }) + + // Create an instance of IPAddressAllocationReconciler + reconciler := &IPAddressAllocationReconciler{} + + // Call SetupWithManager + err := reconciler.SetupWithManager(mgr) + + // Assert no error is returned + assert.NoError(t, err) + + // Check event filters + updateEvent := event.UpdateEvent{ + ObjectOld: &v1alpha1.IPAddressAllocation{ObjectMeta: metav1.ObjectMeta{Generation: 1}}, + ObjectNew: &v1alpha1.IPAddressAllocation{ObjectMeta: metav1.ObjectMeta{Generation: 2}}, + } + deleteEvent := event.DeleteEvent{} + + pred := predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + } + + assert.True(t, pred.Update(updateEvent)) + assert.False(t, pred.Delete(deleteEvent)) + +} diff --git a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller.go b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller.go index ca14aa1e5..ff3e181b5 100644 --- a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller.go +++ b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller.go @@ -215,7 +215,8 @@ func (r *NSXServiceAccountReconciler) validateRealized(count uint16, ca []byte, return count, ca } -func (r *NSXServiceAccountReconciler) garbageCollector(nsxServiceAccountUIDSet sets.Set[string], nsxServiceAccountList *nsxvmwarecomv1alpha1.NSXServiceAccountList) (gcSuccessCount, gcErrorCount uint32) { +func (r *NSXServiceAccountReconciler) garbageCollector(nsxServiceAccountUIDSet sets.Set[string], + nsxServiceAccountList *nsxvmwarecomv1alpha1.NSXServiceAccountList) (gcSuccessCount, gcErrorCount uint32) { nsxServiceAccountCRUIDMap := map[string]types.NamespacedName{} for _, nsxServiceAccount := range nsxServiceAccountList.Items { nsxServiceAccountCRUIDMap[string(nsxServiceAccount.UID)] = types.NamespacedName{ diff --git a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go index 73c4c30a6..47170396d 100644 --- a/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go +++ b/pkg/controllers/nsxserviceaccount/nsxserviceaccount_controller_test.go @@ -875,7 +875,8 @@ func TestNSXServiceAccountReconciler_garbageCollector(t *testing.T) { }) }, args: args{ - nsxServiceAccountUIDSet: sets.New[string]("00000000-0000-0000-0000-000000000002", "00000000-0000-0000-0000-000000000003", "00000000-0000-0000-0000-000000000004"), + nsxServiceAccountUIDSet: sets.New[string]("00000000-0000-0000-0000-000000000002", "00000000-0000-0000-0000-000000000003", + "00000000-0000-0000-0000-000000000004"), nsxServiceAccountList: &nsxvmwarecomv1alpha1.NSXServiceAccountList{Items: []nsxvmwarecomv1alpha1.NSXServiceAccount{{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns1", diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 412b0bd1e..9bc961f36 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -71,19 +71,20 @@ type Client struct { VPCSecurityClient vpcs.SecurityPoliciesClient VPCRuleClient vpc_sp.RulesClient - OrgRootClient nsx_policy.OrgRootClient - ProjectInfraClient projects.InfraClient - VPCClient projects.VpcsClient - IPBlockClient infra.IpBlocksClient - StaticRouteClient vpcs.StaticRoutesClient - NATRuleClient nat.NatRulesClient - VpcGroupClient vpcs.GroupsClient - PortClient subnets.PortsClient - PortStateClient ports.StateClient - IPPoolClient subnets.IpPoolsClient - IPAllocationClient ip_pools.IpAllocationsClient - SubnetsClient vpcs.SubnetsClient - RealizedStateClient realized_state.RealizedEntitiesClient + OrgRootClient nsx_policy.OrgRootClient + ProjectInfraClient projects.InfraClient + VPCClient projects.VpcsClient + IPBlockClient infra.IpBlocksClient + StaticRouteClient vpcs.StaticRoutesClient + NATRuleClient nat.NatRulesClient + VpcGroupClient vpcs.GroupsClient + PortClient subnets.PortsClient + PortStateClient ports.StateClient + IPPoolClient subnets.IpPoolsClient + IPAllocationClient ip_pools.IpAllocationsClient + SubnetsClient vpcs.SubnetsClient + RealizedStateClient realized_state.RealizedEntitiesClient + IPAddressAllocationClient vpcs.IpAddressAllocationsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -205,12 +207,13 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, - NSXChecker: *nsxChecker, - NSXVerChecker: *nsxVersionChecker, - IPPoolClient: ipPoolClient, - IPAllocationClient: ipAllocationClient, - SubnetsClient: subnetsClient, - RealizedStateClient: realizedStateClient, + NSXChecker: *nsxChecker, + NSXVerChecker: *nsxVersionChecker, + IPPoolClient: ipPoolClient, + IPAllocationClient: ipAllocationClient, + SubnetsClient: subnetsClient, + RealizedStateClient: realizedStateClient, + IPAddressAllocationClient: ipAddressAllocationClient, } // NSX version check will be restarted during SecurityPolicy reconcile // So, it's unnecessary to exit even if failed in the first time diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 08d671dff..55f753bfd 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -53,6 +53,9 @@ const ( TagScopeIPPoolCRName string = "nsx-op/ippool_name" TagScopeIPPoolCRUID string = "nsx-op/ippool_uid" TagScopeIPPoolCRType string = "nsx-op/ippool_type" + TagScopeIPAddressAllocationCRName string = "nsx-op/ipaddressallocation_name" + TagScopeIPAddressAllocationCRUID string = "nsx-op/ipaddressallocation_uid" + TagScopeIPAddressAllocationCRType string = "nsx-op/ipaddressallocation_type" TagScopeIPSubnetName string = "nsx-op/ipsubnet_name" TagScopeVMNamespaceUID string = "nsx-op/vm_namespace_uid" TagScopeVMNamespace string = "nsx-op/vm_namespace" @@ -84,14 +87,15 @@ const ( ValueMinorVersion string = "0" ValuePatchVersion string = "0" - GCInterval = 60 * time.Second - RealizeTimeout = 2 * time.Minute - RealizeMaxRetries = 3 - IPPoolFinalizerName = "ippool.nsx.vmware.com/finalizer" - DefaultSNATID = "DEFAULT" - AVISubnetLBID = "_AVI_SUBNET--LB" - IPPoolTypePublic = "Public" - IPPoolTypePrivate = "Private" + GCInterval = 60 * time.Second + RealizeTimeout = 2 * time.Minute + RealizeMaxRetries = 3 + IPPoolFinalizerName = "ippool.nsx.vmware.com/finalizer" + IPAddressAllocationFinalizerName = "ipaddressallocation.nsx.vmware.com/finalizer" + DefaultSNATID = "DEFAULT" + AVISubnetLBID = "_AVI_SUBNET--LB" + IPPoolTypePublic = "Public" + IPPoolTypePrivate = "Private" SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" NetworkPolicyFinalizerName = "networkpolicy.nsx.vmware.com/finalizer" @@ -156,11 +160,12 @@ var ( // ResourceTypeClusterControlPlane is used by NSXServiceAccountController ResourceTypeClusterControlPlane = "clustercontrolplane" // ResourceTypePrincipalIdentity is used by NSXServiceAccountController, and it is MP resource type. - ResourceTypePrincipalIdentity = "principalidentity" - ResourceTypeSubnet = "VpcSubnet" - ResourceTypeIPPool = "IpAddressPool" - ResourceTypeIPPoolBlockSubnet = "IpAddressPoolBlockSubnet" - ResourceTypeNode = "HostTransportNode" + ResourceTypePrincipalIdentity = "principalidentity" + ResourceTypeSubnet = "VpcSubnet" + ResourceTypeIPPool = "IpAddressPool" + ResourceTypeIPAddressAllocation = "ProjectIpAddressAllocation" + ResourceTypeIPPoolBlockSubnet = "IpAddressPoolBlockSubnet" + ResourceTypeNode = "HostTransportNode" ) type Service struct { diff --git a/pkg/nsx/services/ipaddressallocation/builder.go b/pkg/nsx/services/ipaddressallocation/builder.go new file mode 100644 index 000000000..8f08bb628 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/builder.go @@ -0,0 +1,74 @@ +package ipaddressallocation + +import ( + "fmt" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + util2 "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +var ( + Int64 = common.Int64 + String = common.String +) + +const ( + IPADDRESSALLOCATIONPREFIX = "ipa" +) + +func (service *IPAddressAllocationService) BuildIPAddressAllocation(IPAddressAllocation *v1alpha1.IPAddressAllocation) (*model.VpcIpAddressAllocation, error) { + var IPBlock string + var IpBlockPathList []string + + VPCInfo := service.VPCService.ListVPCInfo(IPAddressAllocation.Namespace) + if len(VPCInfo) == 0 { + log.Error(nil, "failed to find VPCInfo for IPAddressAllocation CR", "IPAddressAllocation", IPAddressAllocation.Name, "namespace", IPAddressAllocation.Namespace) + return nil, fmt.Errorf("failed to find VPCInfo for IPAddressAllocation CR %s in namespace %s", IPAddressAllocation.Name, IPAddressAllocation.Namespace) + } + + ipAddressBlockVisibility := util.ToUpper(IPAddressAllocation.Spec.IPAddressBlockVisibility) + if ipAddressBlockVisibility == v1alpha1.IPAddressVisibilityExternal { + IpBlockPathList = VPCInfo[0].ExternalIPv4Blocks + } else if ipAddressBlockVisibility == v1alpha1.IPAddressVisibilityPrivate { + IpBlockPathList = VPCInfo[0].PrivateIpv4Blocks + } + + for _, ipBlockPath := range IpBlockPathList { + if util.Contains(service.ExhaustedIPBlock, ipBlockPath) { + continue + } + IPBlock = ipBlockPath + log.V(2).Info("use ip block path", "ip block path", ipBlockPath) + break + } + + if IPBlock == "" { + log.Error(nil, "all ip blocks are exhausted", "IpBlockPathList", IpBlockPathList) + return nil, util2.IPBlockAllExhaustedError{Desc: "all ip blocks are exhausted"} + } + + return &model.VpcIpAddressAllocation{ + Id: String(service.buildIPAddressAllocationID(IPAddressAllocation)), + DisplayName: String(service.buildIPAddressAllocationName(IPAddressAllocation)), + Tags: service.buildIPAddressAllocationTags(IPAddressAllocation), + IpAddressBlockVisibility: &ipAddressBlockVisibility, + AllocationSize: Int64(int64(IPAddressAllocation.Spec.AllocationSize)), + IpBlock: String(IPBlock), + }, nil +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationID(IPAddressAllocation *v1alpha1.IPAddressAllocation) string { + return util.GenerateID(string(IPAddressAllocation.UID), IPADDRESSALLOCATIONPREFIX, "", "") +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationName(IPAddressAllocation *v1alpha1.IPAddressAllocation) string { + return util.GenerateDisplayName(IPAddressAllocation.ObjectMeta.Name, IPADDRESSALLOCATIONPREFIX, "", "", service.NSXConfig.Cluster) +} + +func (service *IPAddressAllocationService) buildIPAddressAllocationTags(IPAddressAllocation *v1alpha1.IPAddressAllocation) []model.Tag { + return util.BuildBasicTags(service.NSXConfig.Cluster, IPAddressAllocation, "") +} diff --git a/pkg/nsx/services/ipaddressallocation/builder_test.go b/pkg/nsx/services/ipaddressallocation/builder_test.go new file mode 100644 index 000000000..3bbfd156a --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/builder_test.go @@ -0,0 +1,163 @@ +package ipaddressallocation + +import ( + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/vpcclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +type fakeQueryClient struct { +} + +func (qIface *fakeQueryClient) List(_ string, _ *string, _ *string, _ *int64, _ *bool, _ *string) (model.SearchResponse, error) { + cursor := "2" + resultCount := int64(2) + return model.SearchResponse{ + Results: []*data.StructValue{{}}, + Cursor: &cursor, ResultCount: &resultCount, + }, nil +} + +func createService(t *testing.T) (*vpc.VPCService, *gomock.Controller, *mocks.MockVpcsClient) { + config2 := nsx.NewConfig("localhost", "1", "1", []string{}, 10, 3, 20, 20, true, true, true, ratelimiter.AIMD, nil, nil, []string{}) + + cluster, _ := nsx.NewCluster(config2) + rc, _ := cluster.NewRestConnector() + + mockCtrl := gomock.NewController(t) + mockVpcclient := mocks.NewMockVpcsClient(mockCtrl) + k8sClient := mock_client.NewMockClient(mockCtrl) + + vpcStore := &vpc.VPCStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeStaticRouteCRUID: indexFunc}), + BindingType: model.VpcBindingType(), + }} + + service := &vpc.VPCService{ + Service: common.Service{ + Client: k8sClient, + NSXClient: &nsx.Client{ + QueryClient: &fakeQueryClient{}, + VPCClient: mockVpcclient, + RestConnector: rc, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + NSXConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + VpcStore: vpcStore, + VPCNetworkConfigStore: vpc.VPCNetworkInfoStore{ + VPCNetworkConfigMap: map[string]common.VPCNetworkConfigInfo{}, + }, + VPCNSNetworkConfigStore: vpc.VPCNsNetworkConfigStore{ + VPCNSNetworkConfigMap: map[string]string{}, + }, + } + return service, mockCtrl, mockVpcclient +} + +func TestBuildIPAddressAllocation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vpcService, _, _ := createService(t) + ipAllocService := &IPAddressAllocationService{ + VPCService: vpcService, + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + CoeConfig: &config.CoeConfig{ + Cluster: "default", + }, + }, + }, + } + + t.Run("VPCInfo is empty", func(t *testing.T) { + ipAlloc := &v1alpha1.IPAddressAllocation{} + ipAlloc.Namespace = "default" + ipAlloc.Name = "test-ip-alloc" + + patch := gomonkey.ApplyMethod(reflect.TypeOf(ipAllocService.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, _ string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{} + }) + defer patch.Reset() + + result, err := ipAllocService.BuildIPAddressAllocation(ipAlloc) + assert.Nil(t, result) + assert.EqualError(t, err, "failed to find VPCInfo for IPAddressAllocation CR test-ip-alloc in namespace default") + }) + + t.Run("IPAddressBlockVisibility is External and IP blocks are not exhausted", func(t *testing.T) { + ipAlloc := &v1alpha1.IPAddressAllocation{ + Spec: v1alpha1.IPAddressAllocationSpec{ + IPAddressBlockVisibility: v1alpha1.IPAddressVisibilityExternal, + AllocationSize: 1, + }, + } + ipAlloc.Namespace = "default" + ipAlloc.Name = "test-ip-alloc" + ipAlloc.UID = "default" + + patch := gomonkey.ApplyMethod(reflect.TypeOf(ipAllocService.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, _ string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {ExternalIPv4Blocks: []string{"block1"}}, + } + }) + defer patch.Reset() + + result, err := ipAllocService.BuildIPAddressAllocation(ipAlloc) + assert.NotNil(t, result) + assert.Nil(t, err) + assert.Equal(t, "block1", *result.IpBlock) + }) + + t.Run("IPAddressBlockVisibility is Private and all IP blocks are exhausted", func(t *testing.T) { + ipAlloc := &v1alpha1.IPAddressAllocation{ + Spec: v1alpha1.IPAddressAllocationSpec{ + IPAddressBlockVisibility: v1alpha1.IPAddressVisibilityPrivate, + AllocationSize: 1, + }, + } + ipAlloc.Namespace = "default" + ipAlloc.Name = "test-ip-alloc" + ipAlloc.UID = "default" + + patch := gomonkey.ApplyMethod(reflect.TypeOf(ipAllocService.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, _ string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {ExternalIPv4Blocks: []string{"block1", "block2"}}, + } + }) + ipAllocService.ExhaustedIPBlock = []string{"block1", "block2"} + defer patch.Reset() + + result, err := ipAllocService.BuildIPAddressAllocation(ipAlloc) + assert.Nil(t, result) + assert.EqualError(t, err, "all ip blocks are exhausted") + }) +} diff --git a/pkg/nsx/services/ipaddressallocation/compare.go b/pkg/nsx/services/ipaddressallocation/compare.go new file mode 100644 index 000000000..eea336e1c --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/compare.go @@ -0,0 +1,32 @@ +package ipaddressallocation + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +type ( + IpAddressAllocation model.VpcIpAddressAllocation +) + +type Comparable = common.Comparable + +func (iap *IpAddressAllocation) Key() string { + return *iap.Id +} + +func (iap *IpAddressAllocation) Value() data.DataValue { + s := &IpAddressAllocation{Id: iap.Id, DisplayName: iap.DisplayName, Tags: iap.Tags} + dataValue, _ := ComparableToIpAddressAllocation(s).GetDataValue__() + return dataValue +} + +func IpAddressAllocationToComparable(iap *model.VpcIpAddressAllocation) Comparable { + return (*IpAddressAllocation)(iap) +} + +func ComparableToIpAddressAllocation(iap Comparable) *model.VpcIpAddressAllocation { + return (*model.VpcIpAddressAllocation)(iap.(*IpAddressAllocation)) +} diff --git a/pkg/nsx/services/ipaddressallocation/compare_test.go b/pkg/nsx/services/ipaddressallocation/compare_test.go new file mode 100644 index 000000000..7e18ba86b --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/compare_test.go @@ -0,0 +1,52 @@ +package ipaddressallocation + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func TestKey(t *testing.T) { + id := "test-id" + iap := &IpAddressAllocation{Id: &id} + assert.Equal(t, "test-id", iap.Key()) + + iapNil := &IpAddressAllocation{Id: nil} + assert.Panics(t, func() { iapNil.Key() }) +} + +func TestValue(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + iap := &IpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + dataValue := iap.Value() + expectedDataValue, _ := ComparableToIpAddressAllocation(iap).GetDataValue__() + + assert.Equal(t, expectedDataValue, dataValue) +} + +func TestIpAddressAllocationToComparable(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + vpcIap := &model.VpcIpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + comparable := IpAddressAllocationToComparable(vpcIap) + assert.IsType(t, &IpAddressAllocation{}, comparable) +} + +func TestComparableToIpAddressAllocation(t *testing.T) { + id := "test-id" + displayName := "test-display-name" + tags := []model.Tag{{Scope: String("scope"), Tag: String("tag")}} + iap := &IpAddressAllocation{Id: &id, DisplayName: &displayName, Tags: tags} + + vpcIap := ComparableToIpAddressAllocation(iap) + assert.IsType(t, &model.VpcIpAddressAllocation{}, vpcIap) + assert.Equal(t, id, *vpcIap.Id) + assert.Equal(t, displayName, *vpcIap.DisplayName) + assert.Equal(t, tags, vpcIap.Tags) +} diff --git a/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go new file mode 100644 index 000000000..59a65ca8b --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go @@ -0,0 +1,197 @@ +package ipaddressallocation + +import ( + "context" + "fmt" + "sync" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +var ( + log = logger.Log + MarkedForDelete = true + ResourceTypeIPAddressAllocation = common.ResourceTypeIPAddressAllocation +) + +type IPAddressAllocationService struct { + common.Service + ipAddressAllocationStore *IPAddressAllocationStore + ExhaustedIPBlock []string + VPCService common.VPCServiceProvider +} + +func InitializeIPAddressAllocation(service common.Service, vpcService common.VPCServiceProvider) (*IPAddressAllocationService, error) { + wg := sync.WaitGroup{} + wgDone := make(chan bool) + fatalErrors := make(chan error) + + wg.Add(1) + + ipAddressAllocationService := &IPAddressAllocationService{Service: service, VPCService: vpcService} + ipAddressAllocationService.ipAddressAllocationStore = &IPAddressAllocationStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}), + BindingType: model.IpAddressPoolBindingType(), + }} + + tags := []model.Tag{ + {Scope: String(common.TagScopeIPAddressAllocationCRUID)}, + } + go ipAddressAllocationService.InitializeResourceStore(&wg, fatalErrors, ResourceTypeIPAddressAllocation, tags, ipAddressAllocationService.ipAddressAllocationStore) + + go func() { + wg.Wait() + close(wgDone) + }() + select { + case <-wgDone: + break + case err := <-fatalErrors: + close(fatalErrors) + return ipAddressAllocationService, err + } + return ipAddressAllocationService, nil +} + +func (service *IPAddressAllocationService) CreateOrUpdateIPAddressAllocation(obj *v1alpha1.IPAddressAllocation) error { + nsxIPAddressAllocation, err := service.BuildIPAddressAllocation(obj) + if err != nil { + return err + } + existingIPAddressAllocation, err := service.indexedIPAddressAllocation(obj.UID) + if err != nil { + log.Error(err, "failed to get ipaddressallocation", "UID", obj.UID) + return err + } + log.V(1).Info("existing ipaddressallocation", "ipaddressallocation", existingIPAddressAllocation) + ipAddressAllocationUpdated := common.CompareResource(IpAddressAllocationToComparable(existingIPAddressAllocation), + IpAddressAllocationToComparable(nsxIPAddressAllocation)) + + if err := service.Apply(nsxIPAddressAllocation, ipAddressAllocationUpdated); err != nil { + return err + } + + cidr, e := service.acquireCidr(obj, nsxIPAddressAllocation) + if e != nil { + return e + } + if cidr == nil { + return fmt.Errorf("ipaddressallocation %s didn't realize available cidr", obj.UID) + } + obj.Status.CIDR = *cidr + return nil +} + +func (service *IPAddressAllocationService) Apply(nsxIPAddressAllocation *model.VpcIpAddressAllocation, ipAddressAllocationUpdated bool) error { + if !ipAddressAllocationUpdated { + return nil + } + ns := service.GetIPAddressAllocationNamespace(nsxIPAddressAllocation) + var err error + VPCInfo := service.VPCService.ListVPCInfo(ns) + if len(VPCInfo) == 0 { + err = util.NoEffectiveOption{Desc: "no valid org and project for ipaddressallocation"} + } else { + err := service.NSXClient.IPAddressAllocationClient.Patch(VPCInfo[0].OrgID, VPCInfo[0].ProjectID, VPCInfo[0].ID, *nsxIPAddressAllocation.Id, *nsxIPAddressAllocation) + err = util.NSXApiError(err) + if err != nil { + return err + } + } + err = service.ipAddressAllocationStore.Apply(nsxIPAddressAllocation) + if err != nil { + return err + } + log.V(1).Info("successfully created or updated ipaddressallocation", "nsxIPAddressAllocation", nsxIPAddressAllocation) + return nil +} + +func (service *IPAddressAllocationService) DeleteIPAddressAllocation(obj interface{}) error { + var err error + var nsxIPAddressAllocation *model.VpcIpAddressAllocation + switch o := obj.(type) { + case *v1alpha1.IPAddressAllocation: + nsxIPAddressAllocation, err = service.BuildIPAddressAllocation(o) + if err != nil { + log.Error(err, "failed to build ipaddressallocation", "IPAddressAllocation", o) + return err + } + case types.UID: + nsxIPAddressAllocation, err = service.indexedIPAddressAllocation(o) + if err != nil { + log.Error(err, "failed to get ipaddressallocation by UID", "UID", o) + return err + } + } + ns := service.GetIPAddressAllocationNamespace(nsxIPAddressAllocation) + VPCInfo := service.VPCService.ListVPCInfo(ns) + if len(VPCInfo) == 0 { + err = util.NoEffectiveOption{Desc: "no valid org and project for ipaddressallocation"} + } else { + err := service.NSXClient.IPAddressAllocationClient.Delete(VPCInfo[0].OrgID, VPCInfo[0].ProjectID, VPCInfo[0].ID, *nsxIPAddressAllocation.Id) + if err != nil { + return err + } + nsxIPAddressAllocation.MarkedForDelete = &MarkedForDelete + err = service.ipAddressAllocationStore.Apply(nsxIPAddressAllocation) + if err != nil { + return err + } + } + log.V(1).Info("successfully deleted nsxIPAddressAllocation", "nsxIPAddressAllocation", nsxIPAddressAllocation) + return nil +} + +func (service *IPAddressAllocationService) acquireCidr(obj *v1alpha1.IPAddressAllocation, nsxIPAddressAllocation *model.VpcIpAddressAllocation) (*string, error) { + VPCInfo := service.VPCService.ListVPCInfo(obj.Namespace) + var err error + if len(VPCInfo) == 0 { + err = util.NoEffectiveOption{Desc: "no effective org and project for ippool"} + return nil, err + } + m, err := service.NSXClient.IPAddressAllocationClient.Get(VPCInfo[0].OrgID, VPCInfo[0].ProjectID, VPCInfo[0].ID, *nsxIPAddressAllocation.Id) + err = util.NSXApiError(err) + if err != nil { + return nil, err + } + return m.AllocationIps, nil +} + +func (service *IPAddressAllocationService) ListIPAddressAllocationID() sets.Set[string] { //nolint:staticcheck // Ignore the deprecation warning for sets.String + ipAddressAllocationSet := service.ipAddressAllocationStore.ListIndexFuncValues(common.TagScopeIPAddressAllocationCRUID) + return ipAddressAllocationSet +} + +func (service *IPAddressAllocationService) GetIPAddressAllocationNamespace(nsxIPAddressAllocation *model.VpcIpAddressAllocation) string { + for _, tag := range nsxIPAddressAllocation.Tags { + if *tag.Scope == common.TagScopeNamespace { + return *tag.Tag + } + } + return "" +} + +func (service *IPAddressAllocationService) Cleanup(ctx context.Context) error { + uids := service.ListIPAddressAllocationID() + log.Info("cleaning up ipaddressallocation", "count", len(uids)) + for uid := range uids { + select { + case <-ctx.Done(): + return util.TimeoutFailed + default: + err := service.DeleteIPAddressAllocation(types.UID(uid)) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/nsx/services/ipaddressallocation/store.go b/pkg/nsx/services/ipaddressallocation/store.go new file mode 100644 index 000000000..3a0c69632 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/store.go @@ -0,0 +1,89 @@ +package ipaddressallocation + +import ( + "errors" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func keyFunc(obj interface{}) (string, error) { + switch v := obj.(type) { + case *model.VpcIpAddressAllocation: + return *v.Id, nil + case *model.GenericPolicyRealizedResource: + return *v.Id, nil + default: + return "", errors.New("keyFunc doesn't support unknown type") + } +} + +func indexFunc(obj interface{}) ([]string, error) { + res := make([]string, 0, 5) + switch v := obj.(type) { + case *model.VpcIpAddressAllocation: + return filterTag(v.Tags), nil + case *model.GenericPolicyRealizedResource: + return filterTag(v.Tags), nil + default: + return res, errors.New("indexFunc doesn't support unknown type") + } +} + +var filterTag = func(v []model.Tag) []string { + res := make([]string, 0, 5) + for _, tag := range v { + if *tag.Scope == common.TagScopeIPAddressAllocationCRUID { + res = append(res, *tag.Tag) + } + } + return res +} + +type IPAddressAllocationStore struct { + common.ResourceStore +} + +func (ipAddressAllocationStore *IPAddressAllocationStore) Apply(i interface{}) error { + ipAddressAllocation := i.(*model.VpcIpAddressAllocation) + if ipAddressAllocation.MarkedForDelete != nil && *ipAddressAllocation.MarkedForDelete { + err := ipAddressAllocationStore.Delete(ipAddressAllocation) + log.V(1).Info("delete ipAddressAllocation from store", "ipAddressAllocation", ipAddressAllocation) + if err != nil { + return err + } + } else { + err := ipAddressAllocationStore.Add(ipAddressAllocation) + log.V(1).Info("add ipAddressAllocation to store", "ipAddressAllocation", ipAddressAllocation) + if err != nil { + return err + } + } + return nil +} + +func (service *IPAddressAllocationService) indexedIPAddressAllocation(uid types.UID) (*model.VpcIpAddressAllocation, error) { + nsxIPAddressAllocation, err := service.ipAddressAllocationStore.GetByIndex(uid) + if err != nil { + return nil, err + } + return nsxIPAddressAllocation, nil +} + +func (ipAddressAllocationStore *IPAddressAllocationStore) GetByIndex(uid types.UID) (*model.VpcIpAddressAllocation, error) { + nsxIPAddressAllocation := &model.VpcIpAddressAllocation{} + indexResults, err := ipAddressAllocationStore.ResourceStore.ByIndex(common.TagScopeIPAddressAllocationCRUID, string(uid)) + if err != nil { + log.Error(err, "failed to get ipaddressallocation", "UID", string(uid)) + return nil, err + } + if len(indexResults) > 0 { + t := indexResults[0].(*model.VpcIpAddressAllocation) + nsxIPAddressAllocation = t + } else { + log.Info("did not get ipaddressallocation with index", "UID", string(uid)) + } + return nsxIPAddressAllocation, nil +} diff --git a/pkg/nsx/services/ipaddressallocation/store_test.go b/pkg/nsx/services/ipaddressallocation/store_test.go new file mode 100644 index 000000000..002f6a429 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/store_test.go @@ -0,0 +1,98 @@ +package ipaddressallocation + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func TestIPAddressAllocationStore_CRUDResource(t *testing.T) { + ipAddressAllocationCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}) + resourceStore := common.ResourceStore{ + Indexer: ipAddressAllocationCacheIndexer, + BindingType: model.VpcIpAddressAllocationBindingType(), + } + ipAddressAllocationStore := &IPAddressAllocationStore{ResourceStore: resourceStore} + type args struct { + i interface{} + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + {"1", args{i: &model.VpcIpAddressAllocation{Id: String("1")}}, assert.NoError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantErr(t, ipAddressAllocationStore.Apply(tt.args.i), fmt.Sprintf("Apply(%v)", tt.args.i)) + }) + } +} + +func TestIPAddressAllocationStore_GetByIndex(t *testing.T) { + p := &model.VpcIpAddressAllocation{Id: String("1"), DisplayName: String("1"), + Tags: []model.Tag{{Scope: String(common.TagScopeIPAddressAllocationCRUID), + Tag: String("1")}}} + ipAddressAllocationStore := &IPAddressAllocationStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}), + BindingType: model.VpcIpAddressBindingType(), + }} + _ = ipAddressAllocationStore.Apply(p) + type args struct { + uid types.UID + } + tests := []struct { + name string + args args + want *model.VpcIpAddressAllocation + wantErr bool + }{ + {"1", args{uid: "1"}, &model.VpcIpAddressAllocation{Id: String("1"), DisplayName: String("1"), + Tags: []model.Tag{{Scope: String(common.TagScopeIPAddressAllocationCRUID), Tag: String("1")}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ipAddressAllocationStore.GetByIndex(tt.args.uid) + if (err != nil) != tt.wantErr { + t.Errorf("indexedIPAddressAllocation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("indexedIPAddressAllocation() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_indexFunc(t *testing.T) { + mId, mTag, mScope := "11111", "11111", common.TagScopeIPAddressAllocationCRUID + m := &model.VpcIpAddressAllocation{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + } + t.Run("1", func(t *testing.T) { + got, _ := indexFunc(m) + if !reflect.DeepEqual(got, []string{"11111"}) { + t.Errorf("indexFunc() = %v, want %v", got, model.Tag{Tag: &mTag, Scope: &mScope}) + } + }) +} + +func Test_keyFunc(t *testing.T) { + Id := "11111" + g := &model.VpcIpAddressAllocation{Id: &Id} + t.Run("2", func(t *testing.T) { + got, _ := keyFunc(g) + if got != "11111" { + t.Errorf("keyFunc() = %v, want %v", got, "11111") + } + }) +} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 0b537b313..e6060c6fa 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -477,6 +477,10 @@ func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []mo tags = append(tags, model.Tag{Scope: String(common.TagScopeNamespace), Tag: String(i.ObjectMeta.Namespace)}) tags = append(tags, model.Tag{Scope: String(common.TagScopeIPPoolCRName), Tag: String(i.ObjectMeta.Name)}) tags = append(tags, model.Tag{Scope: String(common.TagScopeIPPoolCRUID), Tag: String(string(i.UID))}) + case *v1alpha1.IPAddressAllocation: + tags = append(tags, model.Tag{Scope: String(common.TagScopeNamespace), Tag: String(i.ObjectMeta.Namespace)}) + tags = append(tags, model.Tag{Scope: String(common.TagScopeIPAddressAllocationCRName), Tag: String(i.ObjectMeta.Name)}) + tags = append(tags, model.Tag{Scope: String(common.TagScopeIPAddressAllocationCRUID), Tag: String(string(i.UID))}) default: log.Info("unknown obj type", "obj", obj) }