[VPC] Support NSXLB for VPC

Signed-off-by: gran <gran@vmware.com>

pkg/config/config.go

@@ -417,3 +417,10 @@ func (coeConfig *CoeConfig) validate() error {
 func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error {
 	return nsxConfig.validate(true)
 }
+
+func (nsxConfig *NsxConfig) NSXLBEnabled() bool {
+	if nsxConfig.UseAVILB == false && (nsxConfig.UseNativeLoadBalancer == nil || *nsxConfig.UseNativeLoadBalancer == true) {
+		return true
+	}
+	return false
+}

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
 			return common.ResultRequeue, err
 		}
-		if !isShared {
+		if r.Service.NSXConfig.NsxConfig.UseAVILB && !isShared {
 			err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
 			if err != nil {
 				state := &v1alpha1.VPCState{
@@ -113,10 +113,11 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			}
 		}
 
+		// TODO(gran) check if we have NSXLBSubnetInfo
 		// if lb vpc enabled, read avi subnet path and cidr
 		// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
 		// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
-		if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
+		if r.Service.NSXConfig.NsxConfig.UseAVILB && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
 			path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
 			if err != nil {
 				log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
@@ -139,7 +140,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			LoadBalancerIPAddresses: cidr,
 			PrivateIPv4CIDRs:        nc.PrivateIPv4CIDRs,
 		}
-		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
+		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(nc.Org, nc.NsxtProject, *createdVpc.Id, obj.Namespace))
 	} else {
 		if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
 			metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
 }
 
 func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
-	vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
+	vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
 	setNetworkInfoVPCStatus(c, o, client, vpcState)
 	// ako needs to know the avi subnet path created by nsx
-	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
+	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
 	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
 }
@@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
 	}
 }
 
-func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
+func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
 	// read v1alpha1.VPCNetworkConfiguration by ncName
 	nc := &v1alpha1.VPCNetworkConfiguration{}
 	err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
@@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client
 	createdVPCInfo := &v1alpha1.VPCInfo{
 		Name:            vpcName,
 		AVISESubnetPath: aviSubnetPath,
+		NSXLBSPath:      nsxLBSPath,
 	}
 	// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
 	for i, vpc := range nc.Status.VPCs {

pkg/nsx/client.go

@@ -84,6 +84,7 @@ type Client struct {
 	IPAllocationClient  ip_pools.IpAllocationsClient
 	SubnetsClient       vpcs.SubnetsClient
 	RealizedStateClient realized_state.RealizedEntitiesClient
+	VPCLBSClient        vpcs.VpcLbsClient
 
 	NSXChecker    NSXHealthChecker
 	NSXVerChecker NSXVersionChecker
@@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster))
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
+	vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
 
 	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
 	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		SubnetStatusClient: subnetStatusClient,
 		VPCSecurityClient:  vpcSecurityClient,
 		VPCRuleClient:      vpcRuleClient,
+		VPCLBSClient:       vpcLBSClient,
 
 		NSXChecker:          *nsxChecker,
 		NSXVerChecker:       *nsxVersionChecker,

pkg/nsx/services/common/types.go

@@ -144,6 +144,7 @@ var (
 	ResourceTypeVpc                    = "Vpc"
 	ResourceTypeSubnetPort             = "VpcSubnetPort"
 	ResourceTypeVirtualMachine         = "VirtualMachine"
+	ResourceTypeLBService              = "LBService"
 	ResourceTypeShare                  = "Share"
 	ResourceTypeSharedResource         = "SharedResource"
 	ResourceTypeChildSharedResource    = "ChildSharedResource"

pkg/nsx/services/common/wrap.go

@@ -0,0 +1,84 @@
+package common
+
+import (
+	"github.com/openlyinc/pointy"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+// WrapInfra TODO(gran) refactor existing code in other package
+func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
+	// This is the outermost layer of the hierarchy infra client.
+	// It doesn't need ID field.
+	resourceType := ResourceTypeInfra
+	infraObj := model.Infra{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &infraObj, nil
+}
+
+func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
+	resourceType := ResourceTypeOrgRoot
+	orgRootObj := model.OrgRoot{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &orgRootObj, nil
+}
+
+func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeOrg
+	return wrapChildResourceReference(targetType, org, children)
+}
+
+func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeProject
+	return wrapChildResourceReference(targetType, nsxtProject, children)
+}
+
+func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) {
+	resourceType := ResourceTypeChildResourceReference
+	childProject := model.ChildResourceReference{
+		Id:           &id,
+		ResourceType: resourceType,
+		TargetType:   &targetType,
+		Children:     children,
+	}
+	dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
+	if len(errors) > 0 {
+		return nil, errors[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+
+}
+
+func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
+	vpc.ResourceType = pointy.String(ResourceTypeVpc)
+	childVpc := model.ChildVpc{
+		Id:              vpc.Id,
+		MarkedForDelete: vpc.MarkedForDelete,
+		ResourceType:    "ChildVpc",
+		Vpc:             vpc,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}
+
+func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
+	lbs.ResourceType = pointy.String(ResourceTypeLBService)
+	childLBService := model.ChildLBService{
+		Id:              lbs.Id,
+		MarkedForDelete: lbs.MarkedForDelete,
+		ResourceType:    "ChildLBService",
+		LbService:       lbs,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}

pkg/nsx/services/realizestate/realize_state.go

@@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
 			return err
 		}
 		for _, result := range results.Results {
-			if *result.EntityType != entityType {
+			if entityType != "" && *result.EntityType != entityType {
 				continue
 			}
 			if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {

pkg/nsx/services/vpc/builder.go

@@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace,
 }
 
 func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string,
-	nsxVPC *model.Vpc) (*model.Vpc,
+	nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc,
 	error) {
 	vpc := &model.Vpc{}
 	if nsxVPC != nil {
@@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			return nil, nil
 		}
 		// for updating vpc case, use current vpc id, name
-		vpc = nsxVPC
+		*vpc = *nsxVPC
 	} else {
 		// for creating vpc case, fill in vpc properties based on networkconfig
 		vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
@@ -76,7 +76,9 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			},
 		}
 		vpc.SiteInfos = siteInfos
-		vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
+		if useAVILB {
+			vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
+		}
 		vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
 	}
 
@@ -89,3 +91,16 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 
 	return vpc, nil
 }
+
+func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
+	lbs := &model.LBService{}
+	lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
+	lbs.Id = common.String(string(nsObj.GetUID()))
+	lbs.DisplayName = &lbsName
+	// TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag?
+	lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
+	lbs.Size = &lbsSize
+	lbs.ConnectivityPath = &vpcPath
+	lbs.RelaxScaleValidation = relaxScaleValidation
+	return lbs, nil
+}

pkg/nsx/services/vpc/vpc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"math"
 	"net"
 	"strings"
@@ -30,6 +31,7 @@ const (
 	AviSEIngressAllowRuleId    = "avi-se-ingress-allow-rule"
 	VPCAviSEGroupId            = "avi-se-vms"
 	VpcDefaultSecurityPolicyId = "default-layer3-section"
+	VPCKey                     = "/orgs/%s/projects/%s/vpcs/%s"
 	GroupKey                   = "/orgs/%s/projects/%s/vpcs/%s/groups/%s"
 	SecurityPolicyKey          = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s"
 	RuleKey                    = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s"
@@ -41,8 +43,9 @@ var (
 	ResourceTypeVPC = common.ResourceTypeVpc
 	NewConverter    = common.NewConverter
 
-	MarkedForDelete    = true
-	enableAviAllowRule = false
+	MarkedForDelete           = true
+	enableAviAllowRule        = false
+	EnforceRevisionCheckParam = false
 )
 
 type VPCNetworkInfoStore struct {
@@ -556,7 +559,7 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 		nsxVPC = nil
 	}
 
-	createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC)
+	createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC, s.NSXConfig.NsxConfig.UseAVILB)
 	if err != nil {
 		log.Error(err, "failed to build NSX VPC object")
 		return nil, nil, err
@@ -568,8 +571,33 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 		return existingVPC[0], &nc, nil
 	}
 
+	// build NSX LBS
+	var createdLBS *model.LBService
+	// TODO(gran) use switch to enable/disable NSXLB and AVI
+	if s.NSXConfig.NsxConfig.NSXLBEnabled() {
+		lbsSize := s.NSXConfig.NsxConfig.ServiceSize
+		if lbsSize == "" {
+			lbsSize = config.LB_SERVICE_SIZE_SMALL
+		}
+		if nc.LbServiceSize != "" {
+			lbsSize = nc.LbServiceSize
+		}
+		vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name)
+		var relaxScaleValidation *bool
+		if s.NSXConfig.NsxConfig.RelaxScaleValidaion {
+			relaxScaleValidation = common.Bool(true)
+		}
+		createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation)
+	}
+	// build HAPI request
+	orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NsxtProject, createdVpc, createdLBS)
+	if err != nil {
+		log.Error(err, "failed to build HAPI request")
+		return nil, nil, err
+	}
+
 	log.Info("creating NSX VPC", "VPC", *createdVpc.Id)
-	err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc)
+	err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &EnforceRevisionCheckParam)
 	err = nsxutil.NSXApiError(err)
 	if err != nil {
 		log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace)
@@ -611,6 +639,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 	}
 
 	s.VpcStore.Add(&newVpc)
+
+	// Check LBS realization
+	if createdLBS != nil {
+		newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id)
+		if err != nil {
+			log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id)
+			return nil, nil, err
+		}
+
+		realizeService := realizestate.InitializeRealizeState(s.Service)
+		if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newLBS.Path, ""); err != nil {
+			log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id)
+			if realizestate.IsRealizeStateError(err) {
+				log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id)
+				// delete the nsx vpc object and re-created in next loop
+				if err := s.DeleteVPC(*newVpc.Path); err != nil {
+					log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id)
+					return nil, nil, err
+				}
+			}
+			return nil, nil, err
+		}
+	}
+
 	return &newVpc, &nc, nil
 }
 
@@ -902,3 +954,17 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo {
 	}
 	return VPCInfoList
 }
+
+func (s *VPCService) GetNSXLBSPath(org, nsxtProject, vpcId, ns string) string {
+	nsObj := &v1.Namespace{}
+	// get ns obj
+	if err := s.Client.Get(ctx, types.NamespacedName{Name: ns}, nsObj); err != nil {
+		log.Error(err, "unable to fetch namespace", "name", ns)
+		return ""
+	}
+	vpcLBS, err := s.NSXClient.VPCLBSClient.Get(org, nsxtProject, vpcId, string(nsObj.GetUID()))
+	if err != nil {
+		return ""
+	}
+	return *vpcLBS.Path
+}