Support to create subnetport in subnetset

1. Adding index by subnet CR uid in subnetport store.
2. Adding function GetPortsOfSubnet in service mediator -- moved several functions from
   service to controller to support it.
3. Support to create subnetport in subnetset belongs to shared VPC or current namespace.
4. Updateing vsphere-automation-sdk-go/services/nsxt to 0.9.8 to fix the relization
   error for subnetport.

go.mod

@@ -6,7 +6,7 @@ replace (
 	github.com/vmware/go-vmware-nsxt => github.com/mengdie-song/go-vmware-nsxt v0.0.0-20220921033217-dbd234470e30 // inventory-keeper includes all commits from github.com/gran-vmv/go-vmware-nsxt v0.0.0-20211206034609-cd7ffaf2c996
 	github.com/vmware/vsphere-automation-sdk-go/lib => github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.5.4
 	github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4
-	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5
+	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8
 	github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6
 )
 

pkg/controllers/subnetport/subnetport_controller.go

@@ -10,11 +10,13 @@ import (
 	"os"
 	"reflect"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -84,7 +86,12 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		// 	defaultVMSubnet = true
 		// }
 		old_status := obj.Status.DeepCopy()
-		err := r.Service.CreateOrUpdateSubnetPort(obj)
+		nsxSubnetPath, err := r.GetSubnetPathForSubnetPort(obj)
+		if err != nil {
+			log.Error(err, "failed to get NSX resource path from subnet", "subnetport", obj)
+			return common.ResultRequeue, err
+		}
+		err = r.Service.CreateOrUpdateSubnetPort(obj, nsxSubnetPath)
 		if err != nil {
 			log.Error(err, "failed to create or update NSX subnet port, would retry exponentially", "subnetport", req.NamespacedName)
 			updateFail(r, &ctx, obj, &err)
@@ -140,17 +147,18 @@ func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 func StartSubnetPortController(mgr ctrl.Manager, commonService servicecommon.Service) {
-	subnetPortReconcile := SubnetPortReconciler{
+	subnetPortReconciler := SubnetPortReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
 	}
 	if subnetPortService, err := subnetport.InitializeSubnetPort(commonService); err != nil {
 		log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort")
 		os.Exit(1)
 	} else {
-		subnetPortReconcile.Service = subnetPortService
+		subnetPortReconciler.Service = subnetPortService
+		common.ServiceMediator.SubnetPortService = subnetPortReconciler.Service
 	}
-	if err := subnetPortReconcile.Start(mgr); err != nil {
+	if err := subnetPortReconciler.Start(mgr); err != nil {
 		log.Error(err, "failed to create controller", "controller", "SubnetPort")
 		os.Exit(1)
 	}
@@ -295,3 +303,126 @@ func updateSuccess(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.Subn
 func deleteSuccess(r *SubnetPortReconciler, _ *context.Context, _ *v1alpha1.SubnetPort) {
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort)
 }
+
+func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(obj *v1alpha1.SubnetPort) (string, error) {
+	subnetPath := ""
+	if len(obj.Spec.Subnet) > 0 {
+		subnet := &v1alpha1.Subnet{}
+		namespacedName := types.NamespacedName{
+			Name:      obj.Spec.Subnet,
+			Namespace: obj.Namespace,
+		}
+		if err := r.Client.Get(context.Background(), namespacedName, subnet); err != nil {
+			log.Error(err, "subnet CR not found", "subnet CR", namespacedName)
+			return subnetPath, err
+		}
+		subnetPath = subnet.Status.NSXResourcePath
+		if len(subnetPath) == 0 {
+			err := fmt.Errorf("empty NSX resource path from subnet %s", subnet.Name)
+			return subnetPath, err
+		}
+	} else if len(obj.Spec.SubnetSet) > 0 {
+		subnetSet := &v1alpha1.SubnetSet{}
+		namespacedName := types.NamespacedName{
+			Name:      obj.Spec.SubnetSet,
+			Namespace: obj.Namespace,
+		}
+		if err := r.Client.Get(context.Background(), namespacedName, subnetSet); err != nil {
+			log.Error(err, "subnetSet CR not found", "subnet CR", namespacedName)
+			return subnetPath, err
+		}
+		subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet)
+		if err != nil {
+			return subnetPath, err
+		}
+		return subnetPath, nil
+	} else {
+		subnetSet, err := r.GetDefaultSubnetSet(obj)
+		if err != nil {
+			return "", err
+		}
+		subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet)
+		if err != nil {
+			return subnetPath, err
+		}
+		return subnetPath, nil
+	}
+	return subnetPath, nil
+}
+
+func (r *SubnetPortReconciler) AllocateSubnetFromSubnetSet(subnetPort *v1alpha1.SubnetPort, subnetSet *v1alpha1.SubnetSet) (string, error) {
+	log.Info("allocating Subnet for SubnetPort", "subnetPort", subnetPort.Name, "subnetSet", subnetSet.Name)
+	subnetPath, err := common.ServiceMediator.GetAvailableSubnet(subnetSet)
+	if err != nil {
+		log.Error(err, "failed to allocate Subnet")
+	}
+	log.Info("allocated Subnet for SubnetPort", "subnetPath", subnetPath)
+	return subnetPath, nil
+}
+
+func (r *SubnetPortReconciler) GetDefaultSubnetSet(subnetPort *v1alpha1.SubnetPort) (*v1alpha1.SubnetSet, error) {
+	targetNamespace, _, err := r.getSharedNamespaceAndVpcForNamespace(subnetPort.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	if targetNamespace == "" {
+		log.Info("subnetport's namespace doesn't have shared VPC, searching the default subnetset in the current namespace", "subnetPort.Name", subnetPort.Name, "subnetPort.Namespace", subnetPort.Namespace)
+		targetNamespace = subnetPort.Namespace
+	}
+	subnetSet, err := r.getDefaultSubnetSetByNamespace(subnetPort, targetNamespace)
+	if err != nil {
+		return nil, err
+	}
+	return subnetSet, err
+}
+
+func (r *SubnetPortReconciler) getSharedNamespaceAndVpcForNamespace(namespaceName string) (string, string, error) {
+	sharedNamespaceName := ""
+	sharedVpcName := ""
+	namespace := &v1.Namespace{}
+	namespacedName := types.NamespacedName{Name: namespaceName}
+	if err := r.Client.Get(context.Background(), namespacedName, namespace); err != nil {
+		log.Error(err, "failed to get target namespace during getting VPC for namespace")
+		return "", "", err
+	}
+	// TODO: import "nsx.vmware.com/vpc_name" as the constant from types afer VPC patch is merged.
+	vpcAnnotation, exists := namespace.Annotations["nsx.vmware.com/vpc_name"]
+	if !exists {
+		return "", "", nil
+	}
+	array := strings.Split(vpcAnnotation, "/")
+	if len(array) != 2 {
+		err := fmt.Errorf("invalid annotation value of 'nsx.vmware.com/vpc_name': %s", vpcAnnotation)
+		return "", "", err
+	}
+	sharedNamespaceName, sharedVpcName = array[0], array[1]
+	log.Info("got shared VPC for namespace", "current namespace", namespaceName, "shared VPC", sharedVpcName, "shared namespace", sharedNamespaceName)
+	return sharedNamespaceName, sharedVpcName, nil
+}
+
+func (r *SubnetPortReconciler) getDefaultSubnetSetByNamespace(subnetPort *v1alpha1.SubnetPort, namespace string) (*v1alpha1.SubnetSet, error) {
+	subnetSetList := &v1alpha1.SubnetSetList{}
+	subnetSetSelector := &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			servicecommon.LabelDefaultSubnetSet: servicecommon.ResourceTypeVirtualMachine,
+		},
+	}
+	labelSelector, _ := metav1.LabelSelectorAsSelector(subnetSetSelector)
+	opts := &client.ListOptions{
+		LabelSelector: labelSelector,
+		Namespace:     namespace,
+	}
+	if err := r.Service.Client.List(context.Background(), subnetSetList, opts); err != nil {
+		log.Error(err, "failed to list default subnetset CR", "namespace", subnetPort.Namespace)
+		return nil, err
+	}
+	if len(subnetSetList.Items) == 0 {
+		return nil, errors.New("default subnetset not found")
+	} else if len(subnetSetList.Items) > 1 {
+		return nil, errors.New("multiple default subnetsets found")
+	}
+	subnetSet := subnetSetList.Items[0]
+	log.Info("got default subnetset", "subnetset.Name", subnetSet.Name, "subnetset.uid", subnetSet.UID)
+	return &subnetSet, nil
+
+}

pkg/nsx/services/common/builder.go

@@ -4,7 +4,6 @@
 package common
 
 import (
-	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -43,7 +42,7 @@ func ParseVPCResourcePath(nsxResourcePath string) (VPCResourceInfo, error) {
 	reExp := regexp.MustCompile(`/orgs/([^/]+)/projects/([^/]+)/vpcs/([^/]+)([/\S+]*)`)
 	matches := reExp.FindStringSubmatch(nsxResourcePath)
 	if len(matches) != 5 {
-		err := errors.New("invalid path")
+		err := fmt.Errorf("invalid path '%s'", nsxResourcePath)
 		return info, err
 	}
 	info.OrgID = matches[1]

pkg/nsx/services/common/types.go

@@ -55,6 +55,8 @@ const (
 	NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer"
 	SubnetPortFinalizerName        = "subnetport.nsx.vmware.com/finalizer"
 	RealizeTimeout                 = 2 * time.Minute
+
+	IndexKeySubnetID = "IndexKeySubnetID"
 )
 
 var (

pkg/nsx/services/mediator/mediator.go

@@ -1,11 +1,16 @@
 package mediator
 
 import (
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/securitypolicy"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
+	"github.com/vmware-tanzu/nsx-operator/pkg/util"
 )
 
 var (
@@ -21,6 +26,7 @@ type ServiceMediator struct {
 	*securitypolicy.SecurityPolicyService
 	*vpc.VPCService
 	*subnet.SubnetService
+	*subnetport.SubnetPortService
 }
 
 // ListVPCInfo is a common method, extracting the org, the project, and the vpc string from vpc path of the VPC model.
@@ -40,3 +46,26 @@ func (serviceMediator *ServiceMediator) ListVPCInfo(ns string) []common.VPCResou
 	}
 	return VPCInfoList
 }
+
+// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary.
+func (serviceMediator *ServiceMediator) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) {
+	subnetList := serviceMediator.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(subnetSet.GetUID()))
+	for _, nsxSubnet := range subnetList {
+		portNums := len(serviceMediator.GetPortsOfSubnet(*nsxSubnet.Id))
+		totalIP := int(*nsxSubnet.Ipv4SubnetSize)
+		if len(nsxSubnet.IpAddresses) > 0 {
+			// totalIP will be overrided if IpAddresses are specified.
+			totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses)
+		}
+		if portNums < totalIP-3 {
+			return *nsxSubnet.Path, nil
+		}
+	}
+	log.Info("the existing subnets are not available, creating new subnet", "subnetList", subnetList, "subnetSet.Name", subnetSet.Name, "subnetSet.Namespace", subnetSet.Namespace)
+	return serviceMediator.CreateOrUpdateSubnet(subnetSet, nil)
+}
+
+func (serviceMediator *ServiceMediator) GetPortsOfSubnet(nsxSubnetID string) (ports []model.SegmentPort) {
+	subnetPortList := serviceMediator.SubnetPortStore.GetByIndex(common.IndexKeySubnetID, nsxSubnetID)
+	return subnetPortList
+}

pkg/nsx/services/subnet/builder.go

@@ -2,6 +2,7 @@ package subnet
 
 import (
 	"fmt"
+
 	"github.com/google/uuid"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -23,13 +24,13 @@ func getCluster(service *SubnetService) string {
 func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) (*model.VpcSubnet, error) {
 	tags = append(tags, service.buildBasicTags(obj)...)
 	var nsxSubnet *model.VpcSubnet
+	boolTrue := true
 	switch o := obj.(type) {
 	case *v1alpha1.Subnet:
 		nsxSubnet = &model.VpcSubnet{
 			Id:          String(string(o.GetUID())),
 			AccessMode:  String(string(o.Spec.AccessMode)),
 			DhcpConfig:  service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)),
-			Tags:        tags,
 			DisplayName: String(fmt.Sprintf("%s-%s", obj.GetNamespace(), obj.GetName())),
 		}
 	case *v1alpha1.SubnetSet:
@@ -38,12 +39,17 @@ func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) (
 			Id:          String(fmt.Sprintf("%s-%s", string(o.GetUID()), index)),
 			AccessMode:  String(string(o.Spec.AccessMode)),
 			DhcpConfig:  service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)),
-			Tags:        tags,
 			DisplayName: String(fmt.Sprintf("%s-%s-%s", obj.GetNamespace(), obj.GetName(), index)),
 		}
 	default:
 		return nil, SubnetTypeError
 	}
+	nsxSubnet.Tags = tags
+	nsxSubnet.AdvancedConfig = &model.SubnetAdvancedConfig{
+		StaticIpAllocation: &model.StaticIpAllocation{
+			Enabled: &boolTrue,
+		},
+	}
 	return nsxSubnet, nil
 }
 

pkg/nsx/services/subnet/subnet.go

@@ -15,7 +15,6 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate"
-	"github.com/vmware-tanzu/nsx-operator/pkg/util"
 )
 
 var (
@@ -156,7 +155,7 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, tags []mod
 		}
 	}
 	log.Info("successfully updated nsxSubnet", "nsxSubnet", nsxSubnet)
-	return *nsxSubnet.Id, nil
+	return *nsxSubnet.Path, nil
 }
 
 func (service *SubnetService) DeleteSubnet(nsxSubnet model.VpcSubnet) error {
@@ -237,24 +236,6 @@ func (service *SubnetService) GetIPPoolUsage(subnet *v1alpha1.Subnet) (*model.Po
 	return service.getIPPoolUsage(&nsxSubnets[0])
 }
 
-// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary.
-func (service *SubnetService) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) {
-	subnetList := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(subnetSet.GetUID()))
-	for _, nsxSubnet := range subnetList {
-		// TODO Get port number by subnet ID.
-		portNums := 1 // portNums := commonctl.ServiceMediator.GetPortOfSubnet(nsxSubnet.Id)
-		totalIP := int(*nsxSubnet.Ipv4SubnetSize)
-		if len(nsxSubnet.IpAddresses) > 0 {
-			// totalIP will be overrided if IpAddresses are specified.
-			totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses)
-		}
-		if portNums < totalIP-3 {
-			return *nsxSubnet.Id, nil
-		}
-	}
-	return service.CreateOrUpdateSubnet(subnetSet, nil)
-}
-
 func (service *SubnetService) UpdateSubnetSetStatus(obj *v1alpha1.SubnetSet) error {
 	var subnetInfoList []v1alpha1.SubnetInfo
 	nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(obj.GetUID()))

pkg/nsx/services/subnetport/builder.go

@@ -19,7 +19,7 @@ var (
 	String = common.String
 )
 
-func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort) (*model.SegmentPort, error) {
+func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort, nsxSubnetPath string) (*model.SegmentPort, error) {
 	allocateAddresses := "BOTH"
 	nsxSubnetPortName := fmt.Sprintf("port-%s", obj.Name)
 	nsxSubnetPortID := string(obj.UID)
@@ -28,8 +28,6 @@ func (service *SubnetPortService) buildSubnetPort(obj *v1alpha1.SubnetPort) (*mo
 	if err != nil {
 		return nil, err
 	}
-	// TODO: Get the subnet's NSX resource path from subnet store after it is implemented.
-	nsxSubnetPath, err := service.GetSubnetPathForSubnetPort(obj)
 	nsxSubnetPortPath := fmt.Sprintf("%s/ports/%s", nsxSubnetPath, obj.UID)
 	if err != nil {
 		return nil, err

pkg/nsx/services/subnetport/store.go

@@ -32,14 +32,28 @@ func filterTag(tags []model.Tag, tagScope string) []string {
 	return res
 }
 
-// subnetPortIndexFunc is used to get index of a resource, usually, which is the UID of the CR controller reconciles,
+// subnetPortIndexByCRUID is used to get index of a resource, usually, which is the UID of the CR controller reconciles,
 // index is used to filter out resources which are related to the CR
-func subnetPortIndexFunc(obj interface{}) ([]string, error) {
+func subnetPortIndexByCRUID(obj interface{}) ([]string, error) {
 	switch o := obj.(type) {
 	case model.SegmentPort:
 		return filterTag(o.Tags, common.TagScopeSubnetPortCRUID), nil
 	default:
-		return nil, errors.New("subnetPortIndexFunc doesn't support unknown type")
+		return nil, errors.New("subnetPortIndexByCRUID doesn't support unknown type")
+	}
+}
+
+func subnetPortIndexBySubnetID(obj interface{}) ([]string, error) {
+	switch o := obj.(type) {
+	case model.SegmentPort:
+		vpcInfo, err := common.ParseVPCResourcePath(*o.Path)
+		if err != nil {
+			return nil, err
+		}
+		return []string{vpcInfo.ParentID}, nil
+
+	default:
+		return nil, errors.New("subnetPortIndexBySubnetID doesn't support unknown type")
 	}
 }