[VPC] Support NSXLB for VPC

Signed-off-by: gran <gran@vmware.com>

pkg/config/config.go

@@ -417,3 +417,10 @@ func (coeConfig *CoeConfig) validate() error {
 func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error {
 	return nsxConfig.validate(true)
 }
+
+func (nsxConfig *NsxConfig) NSXLBEnabled() bool {
+	if nsxConfig.UseAVILB == false && (nsxConfig.UseNativeLoadBalancer == nil || *nsxConfig.UseNativeLoadBalancer == true) {
+		return true
+	}
+	return false
+}

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
 			return common.ResultRequeue, err
 		}
-		if !isShared {
+		if r.Service.NSXConfig.NsxConfig.UseAVILB && !isShared {
 			err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
 			if err != nil {
 				state := &v1alpha1.VPCState{
@@ -113,10 +113,11 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			}
 		}
 
+		// TODO(gran) check if we need to query NSXLBSubnetInfo
 		// if lb vpc enabled, read avi subnet path and cidr
 		// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
 		// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
-		if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
+		if r.Service.NSXConfig.NsxConfig.UseAVILB && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
 			path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
 			if err != nil {
 				log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
@@ -139,7 +140,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			LoadBalancerIPAddresses: cidr,
 			PrivateIPv4CIDRs:        nc.PrivateIPv4CIDRs,
 		}
-		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
+		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id))
 	} else {
 		if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
 			metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
 }
 
 func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
-	vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
+	vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
 	setNetworkInfoVPCStatus(c, o, client, vpcState)
 	// ako needs to know the avi subnet path created by nsx
-	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
+	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
 	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
 }
@@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
 	}
 }
 
-func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
+func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
 	// read v1alpha1.VPCNetworkConfiguration by ncName
 	nc := &v1alpha1.VPCNetworkConfiguration{}
 	err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
@@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client
 	createdVPCInfo := &v1alpha1.VPCInfo{
 		Name:            vpcName,
 		AVISESubnetPath: aviSubnetPath,
+		NSXLBSPath:      nsxLBSPath,
 	}
 	// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
 	for i, vpc := range nc.Status.VPCs {

pkg/nsx/client.go

@@ -84,6 +84,7 @@ type Client struct {
 	IPAllocationClient  ip_pools.IpAllocationsClient
 	SubnetsClient       vpcs.SubnetsClient
 	RealizedStateClient realized_state.RealizedEntitiesClient
+	VPCLBSClient        vpcs.VpcLbsClient
 
 	NSXChecker    NSXHealthChecker
 	NSXVerChecker NSXVersionChecker
@@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster))
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
+	vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
 
 	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
 	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		SubnetStatusClient: subnetStatusClient,
 		VPCSecurityClient:  vpcSecurityClient,
 		VPCRuleClient:      vpcRuleClient,
+		VPCLBSClient:       vpcLBSClient,
 
 		NSXChecker:          *nsxChecker,
 		NSXVerChecker:       *nsxVersionChecker,

pkg/nsx/services/common/types.go

@@ -144,6 +144,7 @@ var (
 	ResourceTypeVpc                    = "Vpc"
 	ResourceTypeSubnetPort             = "VpcSubnetPort"
 	ResourceTypeVirtualMachine         = "VirtualMachine"
+	ResourceTypeLBService              = "LBService"
 	ResourceTypeShare                  = "Share"
 	ResourceTypeSharedResource         = "SharedResource"
 	ResourceTypeChildSharedResource    = "ChildSharedResource"

pkg/nsx/services/common/wrap.go

@@ -0,0 +1,84 @@
+package common
+
+import (
+	"github.com/openlyinc/pointy"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+// WrapInfra TODO(gran) refactor existing code in other package
+func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
+	// This is the outermost layer of the hierarchy infra client.
+	// It doesn't need ID field.
+	resourceType := ResourceTypeInfra
+	infraObj := model.Infra{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &infraObj, nil
+}
+
+func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
+	resourceType := ResourceTypeOrgRoot
+	orgRootObj := model.OrgRoot{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &orgRootObj, nil
+}
+
+func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeOrg
+	return wrapChildResourceReference(targetType, org, children)
+}
+
+func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeProject
+	return wrapChildResourceReference(targetType, nsxtProject, children)
+}
+
+func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) {
+	resourceType := ResourceTypeChildResourceReference
+	childProject := model.ChildResourceReference{
+		Id:           &id,
+		ResourceType: resourceType,
+		TargetType:   &targetType,
+		Children:     children,
+	}
+	dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
+	if len(errors) > 0 {
+		return nil, errors[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+
+}
+
+func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
+	vpc.ResourceType = pointy.String(ResourceTypeVpc)
+	childVpc := model.ChildVpc{
+		Id:              vpc.Id,
+		MarkedForDelete: vpc.MarkedForDelete,
+		ResourceType:    "ChildVpc",
+		Vpc:             vpc,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}
+
+func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
+	lbs.ResourceType = pointy.String(ResourceTypeLBService)
+	childLBService := model.ChildLBService{
+		Id:              lbs.Id,
+		MarkedForDelete: lbs.MarkedForDelete,
+		ResourceType:    "ChildLBService",
+		LbService:       lbs,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}

pkg/nsx/services/realizestate/realize_state.go

@@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
 			return err
 		}
 		for _, result := range results.Results {
-			if *result.EntityType != entityType {
+			if entityType != "" && *result.EntityType != entityType {
 				continue
 			}
 			if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {

pkg/nsx/services/vpc/builder.go

@@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace,
 }
 
 func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string,
-	nsxVPC *model.Vpc) (*model.Vpc,
+	nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc,
 	error) {
 	vpc := &model.Vpc{}
 	if nsxVPC != nil {
@@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			return nil, nil
 		}
 		// for updating vpc case, use current vpc id, name
-		vpc = nsxVPC
+		*vpc = *nsxVPC
 	} else {
 		// for creating vpc case, fill in vpc properties based on networkconfig
 		vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
@@ -76,7 +76,11 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			},
 		}
 		vpc.SiteInfos = siteInfos
-		vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
+		loadBalancerVPCEndpointEnabled := false
+		if useAVILB {
+			loadBalancerVPCEndpointEnabled = true
+		}
+		vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled}
 		vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
 	}
 
@@ -89,3 +93,17 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 
 	return vpc, nil
 }
+
+func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
+	lbs := &model.LBService{}
+	lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
+	// LBS id should equal VPC id
+	lbs.Id = common.String(string(nsObj.GetUID()))
+	lbs.DisplayName = &lbsName
+	// TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag?
+	lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
+	lbs.Size = &lbsSize
+	lbs.ConnectivityPath = &vpcPath
+	lbs.RelaxScaleValidation = relaxScaleValidation
+	return lbs, nil
+}

pkg/nsx/services/vpc/store.go

@@ -13,6 +13,8 @@ func keyFunc(obj interface{}) (string, error) {
 	switch v := obj.(type) {
 	case *model.Vpc:
 		return *v.Id, nil
+	case *model.LBService:
+		return *v.Id, nil
 	case *model.IpAddressBlock:
 		return generateIPBlockKey(*v), nil
 	default:
@@ -27,6 +29,8 @@ func indexFunc(obj interface{}) ([]string, error) {
 	switch o := obj.(type) {
 	case *model.Vpc:
 		return filterTag(o.Tags), nil
+	case *model.LBService:
+		return filterTag(o.Tags), nil
 	case *model.IpAddressBlock:
 		return filterTag(o.Tags), nil
 	default:
@@ -148,6 +152,20 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB
 	return block
 }
 
+// LBSStore is a store for LBS
+type LBSStore struct {
+	common.ResourceStore
+}
+
+func (ls *LBSStore) GetByKey(key string) *model.LBService {
+	obj := ls.ResourceStore.GetByKey(key)
+	if obj != nil {
+		lbs := obj.(*model.LBService)
+		return lbs
+	}
+	return nil
+}
+
 // keyFuncAVI is used to get the key of a AVI rule related resource
 func keyFuncAVI(obj interface{}) (string, error) {
 	switch v := obj.(type) {