Merge pull request #269 from heypnus/vpc/subnetset_port_integration

Support to create subnetport in subnetset

go.mod

@@ -6,7 +6,7 @@ replace (
 	github.com/vmware/go-vmware-nsxt => github.com/mengdie-song/go-vmware-nsxt v0.0.0-20220921033217-dbd234470e30 // inventory-keeper includes all commits from github.com/gran-vmv/go-vmware-nsxt v0.0.0-20211206034609-cd7ffaf2c996
 	github.com/vmware/vsphere-automation-sdk-go/lib => github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.5.4
 	github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4
-	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5
+	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8 // update it from 0.9.5 to 0.9.8 to workaround the relization error for subnetport
 	github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6
 )
 

go.sum

@@ -39,6 +39,8 @@ github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4 h1:HfPCC2OxVMZjMgfRF
 github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.5.4/go.mod h1:GqC85noyNzapJN4vIAO9jJ1EKVo3+jCW4/2VTaMvuSg=
 github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5 h1:TfXvc3C/LLrIqkxcJVdOsrDp9NDmq8dRqxxUoUBXZnA=
 github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.5/go.mod h1:r6vXdrxnApdHWL8zxnKrWXinGoP8HEXvMVLxKcarV8c=
+github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8 h1:l8DRN/HDjLyTnePixRhM9Ey5U9RekYXvOcCeOwNwuRk=
+github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.9.8/go.mod h1:r6vXdrxnApdHWL8zxnKrWXinGoP8HEXvMVLxKcarV8c=
 github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6 h1:sFObhyzxRrFCQvvZgVbWwAGyxUvhlom14Axv0Cjix90=
 github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.3.1-0.20221020082725-84e89979deb6/go.mod h1:75a9Rt4zCs3FA+pFi5HIMdNvl+5AHUDMCNblLZiErg4=
 github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=

pkg/controllers/subnetport/subnetport_controller.go

@@ -10,11 +10,13 @@ import (
 	"os"
 	"reflect"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -84,12 +86,21 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		// 	defaultVMSubnet = true
 		// }
 		old_status := obj.Status.DeepCopy()
-		err := r.Service.CreateOrUpdateSubnetPort(obj)
+		nsxSubnetPath, err := r.GetSubnetPathForSubnetPort(obj)
+		if err != nil {
+			log.Error(err, "failed to get NSX resource path from subnet", "subnetport", obj)
+			return common.ResultRequeue, err
+		}
+		err = r.Service.CreateOrUpdateSubnetPort(obj, nsxSubnetPath)
 		if err != nil {
 			log.Error(err, "failed to create or update NSX subnet port, would retry exponentially", "subnetport", req.NamespacedName)
 			updateFail(r, &ctx, obj, &err)
 			return common.ResultRequeue, err
 		}
+		err = r.updateSubnetStatusOnSubnetPort(obj, nsxSubnetPath)
+		if err != nil {
+			log.Error(err, "failed to retrieve subnet status for subnetport", "subnetport", obj, "nsxSubnetPath", nsxSubnetPath)
+		}
 		if reflect.DeepEqual(old_status, obj.Status) {
 			log.Info("status (without conditions) already matched", "new status", obj.Status, "existing status", old_status)
 		} else {
@@ -140,17 +151,18 @@ func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 func StartSubnetPortController(mgr ctrl.Manager, commonService servicecommon.Service) {
-	subnetPortReconcile := SubnetPortReconciler{
+	subnetPortReconciler := SubnetPortReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
 	}
 	if subnetPortService, err := subnetport.InitializeSubnetPort(commonService); err != nil {
 		log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort")
 		os.Exit(1)
 	} else {
-		subnetPortReconcile.Service = subnetPortService
+		subnetPortReconciler.Service = subnetPortService
+		common.ServiceMediator.SubnetPortService = subnetPortReconciler.Service
 	}
-	if err := subnetPortReconcile.Start(mgr); err != nil {
+	if err := subnetPortReconciler.Start(mgr); err != nil {
 		log.Error(err, "failed to create controller", "controller", "SubnetPort")
 		os.Exit(1)
 	}
@@ -295,3 +307,146 @@ func updateSuccess(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.Subn
 func deleteSuccess(r *SubnetPortReconciler, _ *context.Context, _ *v1alpha1.SubnetPort) {
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort)
 }
+
+func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(obj *v1alpha1.SubnetPort) (string, error) {
+	subnetPath := ""
+	if len(obj.Spec.Subnet) > 0 {
+		subnet := &v1alpha1.Subnet{}
+		namespacedName := types.NamespacedName{
+			Name:      obj.Spec.Subnet,
+			Namespace: obj.Namespace,
+		}
+		if err := r.Client.Get(context.Background(), namespacedName, subnet); err != nil {
+			log.Error(err, "subnet CR not found", "subnet CR", namespacedName)
+			return subnetPath, err
+		}
+		subnetPath = subnet.Status.NSXResourcePath
+		if len(subnetPath) == 0 {
+			err := fmt.Errorf("empty NSX resource path from subnet %s", subnet.Name)
+			return subnetPath, err
+		}
+	} else if len(obj.Spec.SubnetSet) > 0 {
+		subnetSet := &v1alpha1.SubnetSet{}
+		namespacedName := types.NamespacedName{
+			Name:      obj.Spec.SubnetSet,
+			Namespace: obj.Namespace,
+		}
+		if err := r.Client.Get(context.Background(), namespacedName, subnetSet); err != nil {
+			log.Error(err, "subnetSet CR not found", "subnet CR", namespacedName)
+			return subnetPath, err
+		}
+		subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet)
+		if err != nil {
+			return subnetPath, err
+		}
+		return subnetPath, nil
+	} else {
+		subnetSet, err := r.GetDefaultSubnetSet(obj)
+		if err != nil {
+			return "", err
+		}
+		subnetPath, err := r.AllocateSubnetFromSubnetSet(obj, subnetSet)
+		if err != nil {
+			return subnetPath, err
+		}
+		return subnetPath, nil
+	}
+	return subnetPath, nil
+}
+
+func (r *SubnetPortReconciler) AllocateSubnetFromSubnetSet(subnetPort *v1alpha1.SubnetPort, subnetSet *v1alpha1.SubnetSet) (string, error) {
+	log.Info("allocating Subnet for SubnetPort", "subnetPort", subnetPort.Name, "subnetSet", subnetSet.Name)
+	subnetPath, err := common.ServiceMediator.GetAvailableSubnet(subnetSet)
+	if err != nil {
+		log.Error(err, "failed to allocate Subnet")
+	}
+	log.Info("allocated Subnet for SubnetPort", "subnetPath", subnetPath)
+	return subnetPath, nil
+}
+
+func (r *SubnetPortReconciler) GetDefaultSubnetSet(subnetPort *v1alpha1.SubnetPort) (*v1alpha1.SubnetSet, error) {
+	targetNamespace, _, err := r.getSharedNamespaceAndVpcForNamespace(subnetPort.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	if targetNamespace == "" {
+		log.Info("subnetport's namespace doesn't have shared VPC, searching the default subnetset in the current namespace", "subnetPort.Name", subnetPort.Name, "subnetPort.Namespace", subnetPort.Namespace)
+		targetNamespace = subnetPort.Namespace
+	}
+	subnetSet, err := r.getDefaultSubnetSetByNamespace(subnetPort, targetNamespace)
+	if err != nil {
+		return nil, err
+	}
+	return subnetSet, err
+}
+
+func (r *SubnetPortReconciler) getSharedNamespaceAndVpcForNamespace(namespaceName string) (string, string, error) {
+	sharedNamespaceName := ""
+	sharedVpcName := ""
+	namespace := &v1.Namespace{}
+	namespacedName := types.NamespacedName{Name: namespaceName}
+	if err := r.Client.Get(context.Background(), namespacedName, namespace); err != nil {
+		log.Error(err, "failed to get target namespace during getting VPC for namespace")
+		return "", "", err
+	}
+	// TODO: import "nsx.vmware.com/vpc_name" as the constant from types afer VPC patch is merged.
+	vpcAnnotation, exists := namespace.Annotations["nsx.vmware.com/vpc_name"]
+	if !exists {
+		return "", "", nil
+	}
+	array := strings.Split(vpcAnnotation, "/")
+	if len(array) != 2 {
+		err := fmt.Errorf("invalid annotation value of 'nsx.vmware.com/vpc_name': %s", vpcAnnotation)
+		return "", "", err
+	}
+	sharedNamespaceName, sharedVpcName = array[0], array[1]
+	log.Info("got shared VPC for namespace", "current namespace", namespaceName, "shared VPC", sharedVpcName, "shared namespace", sharedNamespaceName)
+	return sharedNamespaceName, sharedVpcName, nil
+}
+
+func (r *SubnetPortReconciler) getDefaultSubnetSetByNamespace(subnetPort *v1alpha1.SubnetPort, namespace string) (*v1alpha1.SubnetSet, error) {
+	subnetSetList := &v1alpha1.SubnetSetList{}
+	subnetSetSelector := &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			servicecommon.LabelDefaultSubnetSet: servicecommon.ResourceTypeVirtualMachine,
+		},
+	}
+	labelSelector, _ := metav1.LabelSelectorAsSelector(subnetSetSelector)
+	opts := &client.ListOptions{
+		LabelSelector: labelSelector,
+		Namespace:     namespace,
+	}
+	if err := r.Service.Client.List(context.Background(), subnetSetList, opts); err != nil {
+		log.Error(err, "failed to list default subnetset CR", "namespace", subnetPort.Namespace)
+		return nil, err
+	}
+	if len(subnetSetList.Items) == 0 {
+		return nil, errors.New("default subnetset not found")
+	} else if len(subnetSetList.Items) > 1 {
+		return nil, errors.New("multiple default subnetsets found")
+	}
+	subnetSet := subnetSetList.Items[0]
+	log.Info("got default subnetset", "subnetset.Name", subnetSet.Name, "subnetset.uid", subnetSet.UID)
+	return &subnetSet, nil
+
+}
+
+func (r *SubnetPortReconciler) updateSubnetStatusOnSubnetPort(subnetPort *v1alpha1.SubnetPort, nsxSubnetPath string) error {
+	gateway, netmask, err := r.Service.GetGatewayNetmaskForSubnetPort(subnetPort, nsxSubnetPath)
+	if err != nil {
+		return err
+	}
+	subnetInfo, err := servicecommon.ParseVPCResourcePath(nsxSubnetPath)
+	if err != nil {
+		return err
+	}
+	// For now, we have an asumption that one subnetport only have one IP address
+	subnetPort.Status.IPAddresses[0].Gateway = gateway
+	subnetPort.Status.IPAddresses[0].Netmask = netmask
+	nsxSubnet := common.ServiceMediator.SubnetStore.GetByKey(subnetInfo.ID)
+	if nsxSubnet == nil {
+		return errors.New("NSX subnet not found in store")
+	}
+	subnetPort.Status.LogicalSwitchID = *nsxSubnet.RealizationId
+	return nil
+}

pkg/controllers/subnetport/subnetport_controller_test.go

@@ -101,8 +101,13 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) {
 			return nil
 		})
 	err = errors.New("CreateOrUpdateSubnetPort failed")
+	patchesGetSubnetPathForSubnetPort := gomonkey.ApplyFunc((*SubnetPortReconciler).GetSubnetPathForSubnetPort,
+		func(r *SubnetPortReconciler, obj *v1alpha1.SubnetPort) (string, error) {
+			return "", nil
+		})
+	defer patchesGetSubnetPathForSubnetPort.Reset()
 	patchesCreateOrUpdateSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort,
-		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error {
+		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error {
 			return err
 		})
 	defer patchesCreateOrUpdateSubnetPort.Reset()
@@ -118,7 +123,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) {
 			return nil
 		})
 	patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort,
-		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error {
+		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error {
 			return nil
 		})
 	defer patchesCreateOrUpdateSubnetPort.Reset()
@@ -142,7 +147,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) {
 		})
 	defer patchesDeleteSubnetPort.Reset()
 	patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort,
-		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort) error {
+		func(s *subnetport.SubnetPortService, obj *v1alpha1.SubnetPort, nsxSubnetPath string) error {
 			assert.FailNow(t, "should not be called")
 			return nil
 		})

pkg/nsx/services/common/builder.go

@@ -4,7 +4,6 @@
 package common
 
 import (
-	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -43,7 +42,7 @@ func ParseVPCResourcePath(nsxResourcePath string) (VPCResourceInfo, error) {
 	reExp := regexp.MustCompile(`/orgs/([^/]+)/projects/([^/]+)/vpcs/([^/]+)([/\S+]*)`)
 	matches := reExp.FindStringSubmatch(nsxResourcePath)
 	if len(matches) != 5 {
-		err := errors.New("invalid path")
+		err := fmt.Errorf("invalid path '%s'", nsxResourcePath)
 		return info, err
 	}
 	info.OrgID = matches[1]

pkg/nsx/services/common/types.go

@@ -58,6 +58,8 @@ const (
 	NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer"
 	SubnetPortFinalizerName        = "subnetport.nsx.vmware.com/finalizer"
 	VPCFinalizerName               = "vpc.nsx.vmware.com/finalizer"
+
+	IndexKeySubnetID = "IndexKeySubnetID"
 )
 
 var (

pkg/nsx/services/mediator/mediator.go

@@ -1,11 +1,16 @@
 package mediator
 
 import (
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/securitypolicy"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
+	"github.com/vmware-tanzu/nsx-operator/pkg/util"
 )
 
 var (
@@ -21,6 +26,7 @@ type ServiceMediator struct {
 	*securitypolicy.SecurityPolicyService
 	*vpc.VPCService
 	*subnet.SubnetService
+	*subnetport.SubnetPortService
 }
 
 // ListVPCInfo is a common method, extracting the org, the project, and the vpc string from vpc path of the VPC model.
@@ -46,3 +52,26 @@ func (serviceMediator *ServiceMediator) ListVPCInfo(ns string) []common.VPCResou
 func (m *ServiceMediator) GetVPCNetworkConfigByNamespace(ns string) *vpc.VPCNetworkConfigInfo {
 	return m.VPCService.GetVPCNetworkConfigByNamespace(ns)
 }
+
+// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary.
+func (serviceMediator *ServiceMediator) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) {
+	subnetList := serviceMediator.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(subnetSet.GetUID()))
+	for _, nsxSubnet := range subnetList {
+		portNums := len(serviceMediator.GetPortsOfSubnet(*nsxSubnet.Id))
+		totalIP := int(*nsxSubnet.Ipv4SubnetSize)
+		if len(nsxSubnet.IpAddresses) > 0 {
+			// totalIP will be overrided if IpAddresses are specified.
+			totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses)
+		}
+		if portNums < totalIP-3 {
+			return *nsxSubnet.Path, nil
+		}
+	}
+	log.Info("the existing subnets are not available, creating new subnet", "subnetList", subnetList, "subnetSet.Name", subnetSet.Name, "subnetSet.Namespace", subnetSet.Namespace)
+	return serviceMediator.CreateOrUpdateSubnet(subnetSet, nil)
+}
+
+func (serviceMediator *ServiceMediator) GetPortsOfSubnet(nsxSubnetID string) (ports []model.SegmentPort) {
+	subnetPortList := serviceMediator.SubnetPortStore.GetByIndex(common.IndexKeySubnetID, nsxSubnetID)
+	return subnetPortList
+}

pkg/nsx/services/subnet/builder.go

@@ -2,6 +2,7 @@ package subnet
 
 import (
 	"fmt"
+
 	"github.com/google/uuid"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -23,13 +24,13 @@ func getCluster(service *SubnetService) string {
 func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) (*model.VpcSubnet, error) {
 	tags = append(tags, service.buildBasicTags(obj)...)
 	var nsxSubnet *model.VpcSubnet
+	boolTrue := true
 	switch o := obj.(type) {
 	case *v1alpha1.Subnet:
 		nsxSubnet = &model.VpcSubnet{
 			Id:          String(string(o.GetUID())),
 			AccessMode:  String(string(o.Spec.AccessMode)),
 			DhcpConfig:  service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)),
-			Tags:        tags,
 			DisplayName: String(fmt.Sprintf("%s-%s", obj.GetNamespace(), obj.GetName())),
 		}
 	case *v1alpha1.SubnetSet:
@@ -38,12 +39,17 @@ func (service *SubnetService) buildSubnet(obj client.Object, tags []model.Tag) (
 			Id:          String(fmt.Sprintf("%s-%s", string(o.GetUID()), index)),
 			AccessMode:  String(string(o.Spec.AccessMode)),
 			DhcpConfig:  service.buildDHCPConfig(int64(o.Spec.IPv4SubnetSize - 4)),
-			Tags:        tags,
 			DisplayName: String(fmt.Sprintf("%s-%s-%s", obj.GetNamespace(), obj.GetName(), index)),
 		}
 	default:
 		return nil, SubnetTypeError
 	}
+	nsxSubnet.Tags = tags
+	nsxSubnet.AdvancedConfig = &model.SubnetAdvancedConfig{
+		StaticIpAllocation: &model.StaticIpAllocation{
+			Enabled: &boolTrue,
+		},
+	}
 	return nsxSubnet, nil
 }