From bcf0b675003ef7e4a9f102ae5b716a5c5106ed27 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 9 Aug 2023 09:15:28 +0000 Subject: [PATCH] Support to create subnetport in subnetset 1. Adding index by subnet CR uid in subnetport store. 2. Adding function GetPortsOfSubnet in service mediator -- moved several functions from service to controller to support it. 3. Support to create subnetport in subnetset belongs to shared VPC or current namespace. 4. Updating vsphere-automation-sdk-go/services/nsxt to 0.9.8 to fix the relization error for subnetport. 5. Updating the real value to subnetport.Status.LogicalSwitchID. --- go.mod | 2 +- go.sum | 2 + .../subnetport/subnetport_controller.go | 163 +++++++++++++++++- .../subnetport/subnetport_controller_test.go | 11 +- pkg/nsx/services/common/builder.go | 3 +- pkg/nsx/services/common/types.go | 2 + pkg/nsx/services/mediator/mediator.go | 29 ++++ pkg/nsx/services/subnet/builder.go | 10 +- pkg/nsx/services/subnet/subnet.go | 21 +-- pkg/nsx/services/subnetport/builder.go | 4 +- pkg/nsx/services/subnetport/store.go | 20 ++- pkg/nsx/services/subnetport/subnetport.go | 129 +++----------- 12 files changed, 252 insertions(+), 144 deletions(-) diff --git a/go.mod b/go.mod index 88f9cde6f..5843c7b34 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace ( github.com/vmware/go-vmware-nsxt => github.com/mengdie-song/go-vmware-nsxt v0.0.0-20220921033217-dbd234470e30 // inventory-keeper includes all commits from github.com/gran-vmv/go-vmware-nsxt v0.0.0-20211206034609-cd7ffaf2c996 github.com/vmware/vsphere-automation-sdk-go/lib => github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.5.4 github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4 - github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8 // update it from 0.9.5 to 0.9.8 to workaround the relization error for subnetport github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6 ) diff --git a/go.sum b/go.sum index 712f6b464..3ea6bbf5e 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4 h1:HfPCC2OxVMZjMgfRF github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4/go.mod h1:GqC85noyNzapJN4vIAO9jJ1EKVo3+jCW4/2VTaMvuSg= github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5 h1:TfXvc3C/LLrIqkxcJVdOsrDp9NDmq8dRqxxUoUBXZnA= github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5/go.mod h1:r6vXdrxnApdHWL8zxnKrWXinGoP8HEXvMVLxKcarV8c= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8 h1:l8DRN/HDjLyTnePixRhM9Ey5U9RekYXvOcCeOwNwuRk= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8/go.mod h1:r6vXdrxnApdHWL8zxnKrWXinGoP8HEXvMVLxKcarV8c= github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6 h1:sFObhyzxRrFCQvvZgVbWwAGyxUvhlom14Axv0Cjix90= github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6/go.mod h1:75a9Rt4zCs3FA+pFi5HIMdNvl+5AHUDMCNblLZiErg4= github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index 8715e421e..d8b5b8b46 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -10,11 +10,13 @@ import ( "os" "reflect" "runtime" + "strings" "time" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -84,12 +86,21 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) // defaultVMSubnet = true // } old_status := obj.Status.DeepCopy() - err := r.Service.CreateOrUpdateSubnetPort(obj) + nsxSubnetPath, err := r.GetSubnetPathForSubnetPort(obj) + if err != nil { + log.Error(err, "failed to get NSX resource path from subnet", "subnetport", obj) + return common.ResultRequeue, err + } + err = r.Service.CreateOrUpdateSubnetPort(obj, nsxSubnetPath) if err != nil { log.Error(err, "failed to create or update NSX subnet port, would retry exponentially", "subnetport", req.NamespacedName) updateFail(r, &ctx, obj, &err) return common.ResultRequeue, err } + err = r.updateSubnetStatusOnSubnetPort(obj, nsxSubnetPath) + if err != nil { + log.Error(err, "failed to retrieve subnet status for subnetport", "subnetport", obj, "nsxSubnetPath", nsxSubnetPath) + } if reflect.DeepEqual(old_status, obj.Status) { log.Info("status (without conditions) already matched", "new status", obj.Status, "existing status", old_status) } else { @@ -140,7 +151,7 @@ func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error { } func StartSubnetPortController(mgr ctrl.Manager, commonService servicecommon.Service) { - subnetPortReconcile := SubnetPortReconciler{ + subnetPortReconciler := SubnetPortReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), } @@ -148,9 +159,10 @@ func StartSubnetPortController(mgr ctrl.Manager, commonService servicecommon.Ser log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort") os.Exit(1) } else { - subnetPortReconcile.Service = subnetPortService + subnetPortReconciler.Service = subnetPortService + common.ServiceMediator.SubnetPortService = subnetPortReconciler.Service } - if err := subnetPortReconcile.Start(mgr); err != nil { + if err := subnetPortReconciler.Start(mgr); err != nil { log.Error(err, "failed to create controller", "controller", "SubnetPort") os.Exit(1) } @@ -295,3 +307,146 @@ func updateSuccess(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.Subn func deleteSuccess(r *SubnetPortReconciler, _ *context.Context, _ *v1alpha1.SubnetPort) { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort) } + +func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(obj *v1alpha1.SubnetPort) (string, error) { + subnetPath := "" + if len(obj.Spec.Subnet) > 0 { + subnet := &v1alpha1.Subnet{} + namespacedName := types.NamespacedName{ + Name: obj.Spec.Subnet, + Namespace: obj.Namespace, + } + if err := r.Client.Get(context.Background(), namespacedName, subnet); err != nil { + log.Error(err, "subnet CR not found", "subnet CR", namespacedName) + return subnetPath, err + } + subnetPath = subnet.Status.NSXResourcePath + if len(subnetPath) == 0 { + err := fmt.Errorf("empty NSX resource path from subnet %s", subnet.Name) + return subnetPath, err + } + } else if len(obj.Spec.SubnetSet) > 0 { + subnetSet := &v1alpha1.SubnetSet{} + namespacedName := types.NamespacedName{ + Name: obj.Spec.SubnetSet, + Namespace: obj.Namespace, + } + if err := r.Client.Get(context.Background(), namespacedName, subnetSet); err != nil { + log.Error(err, "subnetSet CR not found", "subnet CR", namespacedName) + return subnetPath, err + } + subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet) + if err != nil { + return subnetPath, err + } + return subnetPath, nil + } else { + subnetSet, err := r.GetDefaultSubnetSet(obj) + if err != nil { + return "", err + } + subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet) + if err != nil { + return subnetPath, err + } + return subnetPath, nil + } + return subnetPath, nil +} + +func (r *SubnetPortReconciler) AllocateSubnetFromSubnetSet(subnetPort *v1alpha1.SubnetPort, subnetSet *v1alpha1.SubnetSet) (string, error) { + log.Info("allocating Subnet for SubnetPort", "subnetPort", subnetPort.Name, "subnetSet", subnetSet.Name) + subnetPath, err := common.ServiceMediator.GetAvailableSubnet(subnetSet) + if err != nil { + log.Error(err, "failed to allocate Subnet") + } + log.Info("allocated Subnet for SubnetPort", "subnetPath", subnetPath) + return subnetPath, nil +} + +func (r *SubnetPortReconciler) GetDefaultSubnetSet(subnetPort *v1alpha1.SubnetPort) (*v1alpha1.SubnetSet, error) { + targetNamespace, _, err := r.getSharedNamespaceAndVpcForNamespace(subnetPort.Namespace) + if err != nil { + return nil, err + } + if targetNamespace == "" { + log.Info("subnetport's namespace doesn't have shared VPC, searching the default subnetset in the current namespace", "subnetPort.Name", subnetPort.Name, "subnetPort.Namespace", subnetPort.Namespace) + targetNamespace = subnetPort.Namespace + } + subnetSet, err := r.getDefaultSubnetSetByNamespace(subnetPort, targetNamespace) + if err != nil { + return nil, err + } + return subnetSet, err +} + +func (r *SubnetPortReconciler) getSharedNamespaceAndVpcForNamespace(namespaceName string) (string, string, error) { + sharedNamespaceName := "" + sharedVpcName := "" + namespace := &v1.Namespace{} + namespacedName := types.NamespacedName{Name: namespaceName} + if err := r.Client.Get(context.Background(), namespacedName, namespace); err != nil { + log.Error(err, "failed to get target namespace during getting VPC for namespace") + return "", "", err + } + // TODO: import "nsx.vmware.com/vpc_name" as the constant from types afer VPC patch is merged. + vpcAnnotation, exists := namespace.Annotations["nsx.vmware.com/vpc_name"] + if !exists { + return "", "", nil + } + array := strings.Split(vpcAnnotation, "/") + if len(array) != 2 { + err := fmt.Errorf("invalid annotation value of 'nsx.vmware.com/vpc_name': %s", vpcAnnotation) + return "", "", err + } + sharedNamespaceName, sharedVpcName = array[0], array[1] + log.Info("got shared VPC for namespace", "current namespace", namespaceName, "shared VPC", sharedVpcName, "shared namespace", sharedNamespaceName) + return sharedNamespaceName, sharedVpcName, nil +} + +func (r *SubnetPortReconciler) getDefaultSubnetSetByNamespace(subnetPort *v1alpha1.SubnetPort, namespace string) (*v1alpha1.SubnetSet, error) { + subnetSetList := &v1alpha1.SubnetSetList{} + subnetSetSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + servicecommon.LabelDefaultSubnetSet: servicecommon.ResourceTypeVirtualMachine, + }, + } + labelSelector, _ := metav1.LabelSelectorAsSelector(subnetSetSelector) + opts := &client.ListOptions{ + LabelSelector: labelSelector, + Namespace: namespace, + } + if err := r.Service.Client.List(context.Background(), subnetSetList, opts); err != nil { + log.Error(err, "failed to list default subnetset CR", "namespace", subnetPort.Namespace) + return nil, err + } + if len(subnetSetList.Items) == 0 { + return nil, errors.New("default subnetset not found") + } else if len(subnetSetList.Items) > 1 { + return nil, errors.New("multiple default subnetsets found") + } + subnetSet := subnetSetList.Items[0] + log.Info("got default subnetset", "subnetset.Name", subnetSet.Name, "subnetset.uid", subnetSet.UID) + return &subnetSet, nil + +} + +func (r *SubnetPortReconciler) updateSubnetStatusOnSubnetPort(subnetPort *v1alpha1.SubnetPort, nsxSubnetPath string) error { + gateway, netmask, err := r.Service.GetGatewayNetmaskForSubnetPort(subnetPort, nsxSubnetPath) + if err != nil { + return err + } + subnetInfo, err := servicecommon.ParseVPCResourcePath(nsxSubnetPath) + if err != nil { + return err + } + // For now, we have an asumption that one subnetport only have one IP address + subnetPort.Status.IPAddresses[0].Gateway = gateway + subnetPort.Status.IPAddresses[0].Netmask = netmask + nsxSubnet := common.ServiceMediator.SubnetStore.GetByKey(subnetInfo.ID) + if nsxSubnet == nil { + return errors.New("NSX subnet not found in store") + } + subnetPort.Status.LogicalSwitchID = *nsxSubnet.RealizationId + return nil +} diff --git a/pkg/controllers/subnetport/subnetport_controller_test.go b/pkg/controllers/subnetport/subnetport_controller_test.go index 84c95f454..56d00bda9 100644 --- a/pkg/controllers/subnetport/subnetport_controller_test.go +++ b/pkg/controllers/subnetport/subnetport_controller_test.go @@ -101,8 +101,13 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return nil }) err = errors.New("CreateOrUpdateSubnetPort failed") + patchesGetSubnetPathForSubnetPort := gomonkey.ApplyFunc((*SubnetPortReconciler).GetSubnetPathForSubnetPort, + func(r *SubnetPortReconciler, obj *v1alpha1.SubnetPort) (string, error) { + return "", nil + }) + defer patchesGetSubnetPathForSubnetPort.Reset() patchesCreateOrUpdateSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort, - func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error { + func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error { return err }) defer patchesCreateOrUpdateSubnetPort.Reset() @@ -118,7 +123,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return nil }) patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort, - func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error { + func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error { return nil }) defer patchesCreateOrUpdateSubnetPort.Reset() @@ -142,7 +147,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { }) defer patchesDeleteSubnetPort.Reset() patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort, - func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error { + func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error { assert.FailNow(t, "should not be called") return nil }) diff --git a/pkg/nsx/services/common/builder.go b/pkg/nsx/services/common/builder.go index fc032b4df..f671f2446 100644 --- a/pkg/nsx/services/common/builder.go +++ b/pkg/nsx/services/common/builder.go @@ -4,7 +4,6 @@ package common import ( - "errors" "fmt" "regexp" "strings" @@ -43,7 +42,7 @@ func ParseVPCResourcePath(nsxResourcePath string) (VPCResourceInfo, error) { reExp := regexp.MustCompile(`/orgs/([^/]+)/projects/([^/]+)/vpcs/([^/]+)([/\S+]*)`) matches := reExp.FindStringSubmatch(nsxResourcePath) if len(matches) != 5 { - err := errors.New("invalid path") + err := fmt.Errorf("invalid path '%s'", nsxResourcePath) return info, err } info.OrgID = matches[1] diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 5b9a4bfac..320d0aaca 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -58,6 +58,8 @@ const ( NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" SubnetPortFinalizerName = "subnetport.nsx.vmware.com/finalizer" VPCFinalizerName = "vpc.nsx.vmware.com/finalizer" + + IndexKeySubnetID = "IndexKeySubnetID" ) var ( diff --git a/pkg/nsx/services/mediator/mediator.go b/pkg/nsx/services/mediator/mediator.go index 7af5a39b4..760f781a2 100644 --- a/pkg/nsx/services/mediator/mediator.go +++ b/pkg/nsx/services/mediator/mediator.go @@ -1,11 +1,16 @@ package mediator import ( + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/securitypolicy" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + "github.com/vmware-tanzu/nsx-operator/pkg/util" ) var ( @@ -21,6 +26,7 @@ type ServiceMediator struct { *securitypolicy.SecurityPolicyService *vpc.VPCService *subnet.SubnetService + *subnetport.SubnetPortService } // ListVPCInfo is a common method, extracting the org, the project, and the vpc string from vpc path of the VPC model. @@ -46,3 +52,26 @@ func (serviceMediator *ServiceMediator) ListVPCInfo(ns string) []common.VPCResou func (m *ServiceMediator) GetVPCNetworkConfigByNamespace(ns string) *vpc.VPCNetworkConfigInfo { return m.VPCService.GetVPCNetworkConfigByNamespace(ns) } + +// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary. +func (serviceMediator *ServiceMediator) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) { + subnetList := serviceMediator.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(subnetSet.GetUID())) + for _, nsxSubnet := range subnetList { + portNums := len(serviceMediator.GetPortsOfSubnet(*nsxSubnet.Id)) + totalIP := int(*nsxSubnet.Ipv4SubnetSize) + if len(nsxSubnet.IpAddresses) > 0 { + // totalIP will be overrided if IpAddresses are specified. + totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses) + } + if portNums < totalIP-3 { + return *nsxSubnet.Path, nil + } + } + log.Info("the existing subnets are not available, creating new subnet", "subnetList", subnetList, "subnetSet.Name", subnetSet.Name, "subnetSet.Namespace", subnetSet.Namespace) + return serviceMediator.CreateOrUpdateSubnet(subnetSet, nil) +} + +func (serviceMediator *ServiceMediator) GetPortsOfSubnet(nsxSubnetID string) (ports []model.SegmentPort) { + subnetPortList := serviceMediator.SubnetPortStore.GetByIndex(common.IndexKeySubnetID, nsxSubnetID) + return subnetPortList +} diff --git a/pkg/nsx/services/subnet/builder.go b/pkg/nsx/services/subnet/builder.go index 32a5a248e..c126d2b59 100644 --- a/pkg/nsx/services/subnet/builder.go +++ b/pkg/nsx/services/subnet/builder.go @@ -2,6 +2,7 @@ package subnet import ( "fmt" + "github.com/google/uuid" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,13 +24,13 @@ func getCluster(service *SubnetService) string { func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) (*model.VpcSubnet, error) { tags = append(tags, service.buildBasicTags(obj)...) var nsxSubnet *model.VpcSubnet + boolTrue := true switch o := obj.(type) { case *v1alpha1.Subnet: nsxSubnet = &model.VpcSubnet{ Id: String(string(o.GetUID())), AccessMode: String(string(o.Spec.AccessMode)), DhcpConfig: service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)), - Tags: tags, DisplayName: String(fmt.Sprintf("%s-%s", obj.GetNamespace(), obj.GetName())), } case *v1alpha1.SubnetSet: @@ -38,12 +39,17 @@ func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) ( Id: String(fmt.Sprintf("%s-%s", string(o.GetUID()), index)), AccessMode: String(string(o.Spec.AccessMode)), DhcpConfig: service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)), - Tags: tags, DisplayName: String(fmt.Sprintf("%s-%s-%s", obj.GetNamespace(), obj.GetName(), index)), } default: return nil, SubnetTypeError } + nsxSubnet.Tags = tags + nsxSubnet.AdvancedConfig = &model.SubnetAdvancedConfig{ + StaticIpAllocation: &model.StaticIpAllocation{ + Enabled: &boolTrue, + }, + } return nsxSubnet, nil } diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index c34f1552f..7f1c6f107 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -15,7 +15,6 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" - "github.com/vmware-tanzu/nsx-operator/pkg/util" ) var ( @@ -156,7 +155,7 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, tags []mod } } log.Info("successfully updated nsxSubnet", "nsxSubnet", nsxSubnet) - return *nsxSubnet.Id, nil + return *nsxSubnet.Path, nil } func (service *SubnetService) DeleteSubnet(nsxSubnet model.VpcSubnet) error { @@ -237,24 +236,6 @@ func (service *SubnetService) GetIPPoolUsage(subnet *v1alpha1.Subnet) (*model.Po return service.getIPPoolUsage(&nsxSubnets[0]) } -// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary. -func (service *SubnetService) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) { - subnetList := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(subnetSet.GetUID())) - for _, nsxSubnet := range subnetList { - // TODO Get port number by subnet ID. - portNums := 1 // portNums := commonctl.ServiceMediator.GetPortOfSubnet(nsxSubnet.Id) - totalIP := int(*nsxSubnet.Ipv4SubnetSize) - if len(nsxSubnet.IpAddresses) > 0 { - // totalIP will be overrided if IpAddresses are specified. - totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses) - } - if portNums < totalIP-3 { - return *nsxSubnet.Id, nil - } - } - return service.CreateOrUpdateSubnet(subnetSet, nil) -} - func (service *SubnetService) UpdateSubnetSetStatus(obj *v1alpha1.SubnetSet) error { var subnetInfoList []v1alpha1.SubnetInfo nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(obj.GetUID())) diff --git a/pkg/nsx/services/subnetport/builder.go b/pkg/nsx/services/subnetport/builder.go index 2d5e3167b..911676f5a 100644 --- a/pkg/nsx/services/subnetport/builder.go +++ b/pkg/nsx/services/subnetport/builder.go @@ -19,7 +19,7 @@ var ( String = common.String ) -func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort) (*model.SegmentPort, error) { +func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort, nsxSubnetPath string) (*model.SegmentPort, error) { allocateAddresses := "BOTH" nsxSubnetPortName := fmt.Sprintf("port-%s", obj.Name) nsxSubnetPortID := string(obj.UID) @@ -28,8 +28,6 @@ func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort) (*mo if err != nil { return nil, err } - // TODO: Get the subnet's NSX resource path from subnet store after it is implemented. - nsxSubnetPath, err := service.GetSubnetPathForSubnetPort(obj) nsxSubnetPortPath := fmt.Sprintf("%s/ports/%s", nsxSubnetPath, obj.UID) if err != nil { return nil, err diff --git a/pkg/nsx/services/subnetport/store.go b/pkg/nsx/services/subnetport/store.go index 01b83bc7e..547292014 100644 --- a/pkg/nsx/services/subnetport/store.go +++ b/pkg/nsx/services/subnetport/store.go @@ -32,14 +32,28 @@ func filterTag(tags []model.Tag, tagScope string) []string { return res } -// subnetPortIndexFunc is used to get index of a resource, usually, which is the UID of the CR controller reconciles, +// subnetPortIndexByCRUID is used to get index of a resource, usually, which is the UID of the CR controller reconciles, // index is used to filter out resources which are related to the CR -func subnetPortIndexFunc(obj interface{}) ([]string, error) { +func subnetPortIndexByCRUID(obj interface{}) ([]string, error) { switch o := obj.(type) { case model.SegmentPort: return filterTag(o.Tags, common.TagScopeSubnetPortCRUID), nil default: - return nil, errors.New("subnetPortIndexFunc doesn't support unknown type") + return nil, errors.New("subnetPortIndexByCRUID doesn't support unknown type") + } +} + +func subnetPortIndexBySubnetID(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case model.SegmentPort: + vpcInfo, err := common.ParseVPCResourcePath(*o.Path) + if err != nil { + return nil, err + } + return []string{vpcInfo.ParentID}, nil + + default: + return nil, errors.New("subnetPortIndexBySubnetID doesn't support unknown type") } } diff --git a/pkg/nsx/services/subnetport/subnetport.go b/pkg/nsx/services/subnetport/subnetport.go index e96799400..4316d09e7 100644 --- a/pkg/nsx/services/subnetport/subnetport.go +++ b/pkg/nsx/services/subnetport/subnetport.go @@ -4,23 +4,19 @@ package subnetport import ( - "context" "errors" - "fmt" "strings" "sync" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/nsx-operator/pkg/logger" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/pkg/util" @@ -30,17 +26,17 @@ import ( var ( log = logger.Log - ResourceTypeSubnetPort = common.ResourceTypeSubnetPort + ResourceTypeSubnetPort = servicecommon.ResourceTypeSubnetPort MarkedForDelete = true ) type SubnetPortService struct { - common.Service + servicecommon.Service SubnetPortStore *SubnetPortStore } // InitializeSubnetPort sync NSX resources. -func InitializeSubnetPort(service common.Service) (*SubnetPortService, error) { +func InitializeSubnetPort(service servicecommon.Service) (*SubnetPortService, error) { wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) @@ -49,8 +45,13 @@ func InitializeSubnetPort(service common.Service) (*SubnetPortService, error) { subnetPortService := &SubnetPortService{Service: service} - subnetPortService.SubnetPortStore = &SubnetPortStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeSubnetPortCRUID: subnetPortIndexFunc}), + subnetPortService.SubnetPortStore = &SubnetPortStore{ResourceStore: servicecommon.ResourceStore{ + Indexer: cache.NewIndexer( + keyFunc, + cache.Indexers{ + servicecommon.TagScopeSubnetPortCRUID: subnetPortIndexByCRUID, + servicecommon.IndexKeySubnetID: subnetPortIndexBySubnetID, + }), BindingType: model.SegmentPortBindingType(), }} @@ -72,28 +73,21 @@ func InitializeSubnetPort(service common.Service) (*SubnetPortService, error) { return subnetPortService, nil } -func (service *SubnetPortService) CreateOrUpdateSubnetPort(obj *v1alpha1.SubnetPort) error { +func (service *SubnetPortService) CreateOrUpdateSubnetPort(obj *v1alpha1.SubnetPort, nsxSubnetPath string) error { log.Info("Creating or updating subnetport", "subnetport", obj) - - nsxSubnetPath, err := service.GetSubnetPathForSubnetPort(obj) - if err != nil { - log.Error(err, "failed to get NSX resource path from subnet", "subnetport", obj) - return err - } - - nsxSubnetPort, err := service.buildSubnetPort(obj) + nsxSubnetPort, err := service.buildSubnetPort(obj, nsxSubnetPath) if err != nil { log.Error(err, "failed to build NSX subnet port") return err } - subnetInfo, err := common.ParseVPCResourcePath(nsxSubnetPath) + subnetInfo, err := servicecommon.ParseVPCResourcePath(nsxSubnetPath) if err != nil { log.Error(err, "failed to parse NSX subnet path", "path", nsxSubnetPath) } existingSubnetPort := service.SubnetPortStore.GetByKey(*nsxSubnetPort.Id) - isChanged := common.CompareResource(SubnetPortToComparable(existingSubnetPort), SubnetPortToComparable(nsxSubnetPort)) + isChanged := servicecommon.CompareResource(SubnetPortToComparable(existingSubnetPort), SubnetPortToComparable(nsxSubnetPort)) if !isChanged { log.Info("NSX subnet port not changed, skipping the update", "nsxSubnetPort.Id", nsxSubnetPort.Id) // We don't need to update it but still need to check realized state. @@ -113,7 +107,7 @@ func (service *SubnetPortService) CreateOrUpdateSubnetPort(obj *v1alpha1.SubnetP log.Info("created NSX subnet port", "subnetPort", nsxSubnetPort) } } - if err := service.CheckSubnetPortState(obj); err != nil { + if err := service.CheckSubnetPortState(obj, nsxSubnetPath); err != nil { log.Error(err, "check and update NSX subnet port state failed, would retry exponentially", "subnetport", obj.UID) return err } @@ -122,7 +116,7 @@ func (service *SubnetPortService) CreateOrUpdateSubnetPort(obj *v1alpha1.SubnetP } // CheckSubnetPortState will check the port realized status then get the port state to prepare the CR status. -func (service *SubnetPortService) CheckSubnetPortState(obj *v1alpha1.SubnetPort) error { +func (service *SubnetPortService) CheckSubnetPortState(obj *v1alpha1.SubnetPort, nsxSubnetPath string) error { nsxSubnetPort := service.SubnetPortStore.GetByKey(string(obj.UID)) if nsxSubnetPort == nil { return errors.New("failed to get subnet port from store") @@ -141,95 +135,22 @@ func (service *SubnetPortService) CheckSubnetPortState(obj *v1alpha1.SubnetPort) return err } // TODO: avoid to get subnetport state again if we already got it. - nsxPortState, err := service.GetSubnetPortState(obj) + nsxPortState, err := service.GetSubnetPortState(obj, nsxSubnetPath) if err != nil { return err } log.Info("Got the NSX subnet port state", "nsxPortState.RealizedBindings", nsxPortState.RealizedBindings) - gateway, netmask, err := service.getGatewayNetmaskForSubnetPort(obj) - if err != nil { - return err - } ipAddress := v1alpha1.SubnetPortIPAddress{ - Gateway: gateway, - IP: *nsxPortState.RealizedBindings[0].Binding.IpAddress, - Netmask: netmask, + IP: *nsxPortState.RealizedBindings[0].Binding.IpAddress, } obj.Status.IPAddresses = []v1alpha1.SubnetPortIPAddress{ipAddress} obj.Status.MACAddress = strings.Trim(*nsxPortState.RealizedBindings[0].Binding.MacAddress, "\"") obj.Status.VIFID = *nsxPortState.Attachment.Id - // TODO: get the LSP ID when the subnet implementation is ready. - obj.Status.LogicalSwitchID = "b0b6a38b-942f-4029-8f2e-9987ad613197(fake)" - return nil } -func (service *SubnetPortService) GetSubnetPathForSubnetPort(obj *v1alpha1.SubnetPort) (string, error) { - // TODO: For now, only the parent subnet is supported. - path := "" - if len(obj.Spec.Subnet) > 0 { - subnet := &v1alpha1.Subnet{} - namespacedName := types.NamespacedName{ - Name: obj.Spec.Subnet, - Namespace: obj.Namespace, - } - // TODO: Get the subnet's NSX resource path from subnet store after it is implemented. - if err := service.Client.Get(context.Background(), namespacedName, subnet); err != nil { - log.Error(err, "subnet CR not found", "subnet CR", namespacedName) - return path, err - } - path = subnet.Status.NSXResourcePath - if len(path) == 0 { - err := fmt.Errorf("empty NSX resource path from subnet %s", subnet.Name) - return path, err - } - } else if len(obj.Spec.SubnetSet) > 0 { - // TODO: subnetport for VM under subnetset will be implemented after subnetset code is ready - return "", errors.New("subnetport under subnetset is not implemented yet") - } else { - subnetset, err := service.GetDefaultSubnetSet(obj) - if err != nil { - return "", err - } - log.Info("Got default subnetset", "subnetset.Name", subnetset.Name, "subnetset.uid", subnetset.UID) - // TODO: subnetport for VM under subnetset will be implemented after subnetset code is ready - return "", errors.New("subnetport under subnetset is not implemented yet") - } - return path, nil -} - -func (service *SubnetPortService) GetDefaultSubnetSet(obj *v1alpha1.SubnetPort) (*v1alpha1.SubnetSet, error) { - subnetSetList := &v1alpha1.SubnetSetList{} - subnetSetSelector := &metav1.LabelSelector{ - MatchLabels: map[string]string{ - common.LabelDefaultSubnetSet: common.ResourceTypeVirtualMachine, - }, - } - labelSelector, _ := metav1.LabelSelectorAsSelector(subnetSetSelector) - opts := &client.ListOptions{ - LabelSelector: labelSelector, - Namespace: obj.Namespace, - } - if err := service.Client.List(context.Background(), subnetSetList, opts); err != nil { - log.Error(err, "failed to list default subnetset CR", "namespace", obj.Namespace) - return nil, err - } - if len(subnetSetList.Items) == 0 { - return nil, errors.New("default subnetset not found") - } else if len(subnetSetList.Items) > 1 { - return nil, errors.New("multiple default subnetsets found") - } - return &subnetSetList.Items[0], nil -} - -func (service *SubnetPortService) GetSubnetPortState(obj *v1alpha1.SubnetPort) (*model.SegmentPortState, error) { - // TODO: For now, only the parent subnet is supported. - nsxSubnetPath, err := service.GetSubnetPathForSubnetPort(obj) - if err != nil { - return nil, err - } - +func (service *SubnetPortService) GetSubnetPortState(obj *v1alpha1.SubnetPort, nsxSubnetPath string) (*model.SegmentPortState, error) { nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID := nsxutil.ParseVPCPath(nsxSubnetPath) nsxSubnetPortState, err := service.NSXClient.PortStateClient.Get(nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID, string(obj.UID), nil, nil) if err != nil { @@ -260,17 +181,13 @@ func (service *SubnetPortService) DeleteSubnetPort(uid types.UID) error { func (service *SubnetPortService) ListNSXSubnetPortIDForCR() sets.String { log.V(2).Info("listing subnet port CR UIDs") - subnetPortSet := service.SubnetPortStore.ListIndexFuncValues(common.TagScopeSubnetPortCRUID) + subnetPortSet := service.SubnetPortStore.ListIndexFuncValues(servicecommon.TagScopeSubnetPortCRUID) return subnetPortSet } -func (service *SubnetPortService) getGatewayNetmaskForSubnetPort(obj *v1alpha1.SubnetPort) (string, string, error) { - nsxSubnetPath, err := service.GetSubnetPathForSubnetPort(obj) - if err != nil { - return "", "", err - } +func (service *SubnetPortService) GetGatewayNetmaskForSubnetPort(obj *v1alpha1.SubnetPort, nsxSubnetPath string) (string, string, error) { // TODO: merge the logic to subnet service when subnet implementation is done. - subnetInfo, err := common.ParseVPCResourcePath(nsxSubnetPath) + subnetInfo, err := servicecommon.ParseVPCResourcePath(nsxSubnetPath) if err != nil { return "", "", err }