[VPC] Support NSXLB for VPC

Signed-off-by: gran <gran@vmware.com>

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -66,6 +66,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			log.V(1).Info("added finalizer on NetworkInfo CR", "NetworkInfo", req.NamespacedName)
 		}
 
+		// TODO: return NSXLB path here and store to VPCState, then update to VPCInfo
 		createdVpc, nc, err := r.Service.CreateOrUpdateVPC(obj)
 		if err != nil {
 			log.Error(err, "create vpc failed, would retry exponentially", "VPC", req.NamespacedName)

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client
 	createdVPCInfo := &v1alpha1.VPCInfo{
 		Name:            vpcName,
 		AVISESubnetPath: aviSubnetPath,
+		NSXLBSPath:      nsxlbsPath,
 	}
 	// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
 	for i, vpc := range nc.Status.VPCs {

pkg/nsx/services/common/types.go

@@ -144,6 +144,7 @@ var (
 	ResourceTypeVpc                    = "Vpc"
 	ResourceTypeSubnetPort             = "VpcSubnetPort"
 	ResourceTypeVirtualMachine         = "VirtualMachine"
+	ResourceTypeLBService              = "LBService"
 	ResourceTypeShare                  = "Share"
 	ResourceTypeSharedResource         = "SharedResource"
 	ResourceTypeChildSharedResource    = "ChildSharedResource"

pkg/nsx/services/common/wrap.go

@@ -0,0 +1,53 @@
+package common
+
+import (
+	"github.com/openlyinc/pointy"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+// WrapInfra TODO: refactor existing code in other package
+func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
+	// This is the outermost layer of the hierarchy infra client.
+	// It doesn't need ID field.
+	resourceType := ResourceTypeInfra
+	infraObj := model.Infra{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &infraObj, nil
+}
+
+func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
+	var vpcChildren []*data.StructValue
+	vpc.ResourceType = pointy.String(ResourceTypeVpc)
+	childVpc := model.ChildVpc{
+		Id:              vpc.Id,
+		MarkedForDelete: vpc.MarkedForDelete,
+		ResourceType:    "ChildVpc",
+		Vpc:             vpc,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	vpcChildren = append(vpcChildren, dataValue.(*data.StructValue))
+	return vpcChildren, nil
+}
+
+func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
+	var lbServiceChildren []*data.StructValue
+	lbs.ResourceType = pointy.String(ResourceTypeLBService)
+	childLBService := model.ChildLBService{
+		Id:              lbs.Id,
+		MarkedForDelete: lbs.MarkedForDelete,
+		ResourceType:    "ChildLBService",
+		LbService:       lbs,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	lbServiceChildren = append(lbServiceChildren, dataValue.(*data.StructValue))
+	return lbServiceChildren, nil
+}

pkg/nsx/services/vpc/builder.go

@@ -89,3 +89,17 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 
 	return vpc, nil
 }
+
+func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
+	lbs := &model.LBService{}
+	lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
+	lbs.Id = common.String(string(nsObj.GetUID()))
+	lbs.DisplayName = &lbsName
+	// TODO: do we need "created_for" and "lb_t1_link_ip" tag?
+	lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
+	lbs.Size = &lbsSize
+	// TODO: check if ConnectivityPath is expected
+	lbs.ConnectivityPath = &vpcPath
+	lbs.RelaxScaleValidation = relaxScaleValidation
+	return lbs, nil
+}

pkg/nsx/services/vpc/vpc.go

@@ -30,6 +30,7 @@ const (
 	AviSEIngressAllowRuleId    = "avi-se-ingress-allow-rule"
 	VPCAviSEGroupId            = "avi-se-vms"
 	VpcDefaultSecurityPolicyId = "default-layer3-section"
+	VPCKey                     = "/orgs/%s/projects/%s/vpcs/%s"
 	GroupKey                   = "/orgs/%s/projects/%s/vpcs/%s/groups/%s"
 	SecurityPolicyKey          = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s"
 	RuleKey                    = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s"
@@ -41,8 +42,9 @@ var (
 	ResourceTypeVPC = common.ResourceTypeVpc
 	NewConverter    = common.NewConverter
 
-	MarkedForDelete    = true
-	enableAviAllowRule = false
+	MarkedForDelete           = true
+	enableAviAllowRule        = false
+	EnforceRevisionCheckParam = false
 )
 
 type VPCNetworkInfoStore struct {
@@ -558,8 +560,29 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 		return existingVPC[0], &nc, nil
 	}
 
+	// build NSX LBS
+	var createdLBS *model.LBService
+	// TODO: use switch to enable/disable NSXLB and AVI
+	if nsxVPC == nil {
+		lbsSize := string(s.NSXConfig.NsxConfig.ServiceSize)
+		// TODO: read lbsSize from VPC
+		vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name)
+		var relaxScaleValidation *bool
+		if s.NSXConfig.NsxConfig.RelaxScaleValidaion {
+			relaxScaleValidation = common.Bool(true)
+		}
+		createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation)
+	}
+	// build HAPI request
+	infra, err := s.WrapHierarchyVPC(createdVpc, createdLBS)
+	if err != nil {
+		log.Error(err, "failed to build HAPI request")
+		return nil, nil, err
+	}
+
 	log.Info("creating NSX VPC", "VPC", *createdVpc.Id)
-	err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc)
+	err = s.NSXClient.ProjectInfraClient.Patch(nc.Org, nc.NsxtProject, *infra, &EnforceRevisionCheckParam)
+	// TODO: LBService with VPC is not supported by current sdk
 	if err != nil {
 		log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace)
 		// TODO: this seems to be a nsx bug, in some case, even if nsx returns failed but the object is still created.

pkg/nsx/services/vpc/wrap.go

@@ -0,0 +1,29 @@
+package vpc
+
+import (
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+func (s *VPCService) WrapHierarchyVPC(vpc *model.Vpc, lbs *model.LBService) (*model.Infra, error) {
+	var infraChildren []*data.StructValue
+	vpcChildren, err := s.WrapVPC(vpc)
+	if err != nil {
+		return nil, err
+	}
+	infraChildren = append(infraChildren, vpcChildren...)
+	// TODO: check if this relationship is correct
+	if lbs != nil {
+		lbsChildren, err := s.WrapLBS(lbs)
+		if err != nil {
+			return nil, err
+		}
+		infraChildren = append(infraChildren, lbsChildren...)
+	}
+
+	infra, err := s.WrapInfra(infraChildren)
+	if err != nil {
+		return nil, err
+	}
+	return infra, nil
+}