[VPC] Support NSXLB for VPC

Signed-off-by: gran <gran@vmware.com>

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			LoadBalancerIPAddresses: cidr,
 			PrivateIPv4CIDRs:        nc.PrivateIPv4CIDRs,
 		}
-		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
+		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(nc.Org, nc.NsxtProject, *createdVpc.Id, obj.Namespace))
 	} else {
 		if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
 			metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
 }
 
 func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
-	vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
+	vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
 	setNetworkInfoVPCStatus(c, o, client, vpcState)
 	// ako needs to know the avi subnet path created by nsx
-	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
+	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
 	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
 }
@@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
 	}
 }
 
-func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
+func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
 	// read v1alpha1.VPCNetworkConfiguration by ncName
 	nc := &v1alpha1.VPCNetworkConfiguration{}
 	err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
@@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client
 	createdVPCInfo := &v1alpha1.VPCInfo{
 		Name:            vpcName,
 		AVISESubnetPath: aviSubnetPath,
+		NSXLBSPath:      nsxLBSPath,
 	}
 	// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
 	for i, vpc := range nc.Status.VPCs {

pkg/nsx/client.go

@@ -84,6 +84,7 @@ type Client struct {
 	IPAllocationClient  ip_pools.IpAllocationsClient
 	SubnetsClient       vpcs.SubnetsClient
 	RealizedStateClient realized_state.RealizedEntitiesClient
+	VPCLBSClient        vpcs.VpcLbsClient
 
 	NSXChecker    NSXHealthChecker
 	NSXVerChecker NSXVersionChecker
@@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster))
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
+	vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
 
 	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
 	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		SubnetStatusClient: subnetStatusClient,
 		VPCSecurityClient:  vpcSecurityClient,
 		VPCRuleClient:      vpcRuleClient,
+		VPCLBSClient:       vpcLBSClient,
 
 		NSXChecker:          *nsxChecker,
 		NSXVerChecker:       *nsxVersionChecker,

pkg/nsx/services/common/types.go

@@ -144,6 +144,7 @@ var (
 	ResourceTypeVpc                    = "Vpc"
 	ResourceTypeSubnetPort             = "VpcSubnetPort"
 	ResourceTypeVirtualMachine         = "VirtualMachine"
+	ResourceTypeLBService              = "LBService"
 	ResourceTypeShare                  = "Share"
 	ResourceTypeSharedResource         = "SharedResource"
 	ResourceTypeChildSharedResource    = "ChildSharedResource"

pkg/nsx/services/common/wrap.go

@@ -0,0 +1,53 @@
+package common
+
+import (
+	"github.com/openlyinc/pointy"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+// WrapInfra TODO(gran) refactor existing code in other package
+func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
+	// This is the outermost layer of the hierarchy infra client.
+	// It doesn't need ID field.
+	resourceType := ResourceTypeInfra
+	infraObj := model.Infra{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &infraObj, nil
+}
+
+func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
+	var vpcChildren []*data.StructValue
+	vpc.ResourceType = pointy.String(ResourceTypeVpc)
+	childVpc := model.ChildVpc{
+		Id:              vpc.Id,
+		MarkedForDelete: vpc.MarkedForDelete,
+		ResourceType:    "ChildVpc",
+		Vpc:             vpc,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	vpcChildren = append(vpcChildren, dataValue.(*data.StructValue))
+	return vpcChildren, nil
+}
+
+func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
+	var lbServiceChildren []*data.StructValue
+	lbs.ResourceType = pointy.String(ResourceTypeLBService)
+	childLBService := model.ChildLBService{
+		Id:              lbs.Id,
+		MarkedForDelete: lbs.MarkedForDelete,
+		ResourceType:    "ChildLBService",
+		LbService:       lbs,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	lbServiceChildren = append(lbServiceChildren, dataValue.(*data.StructValue))
+	return lbServiceChildren, nil
+}

pkg/nsx/services/realizestate/realize_state.go

@@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
 			return err
 		}
 		for _, result := range results.Results {
-			if *result.EntityType != entityType {
+			if entityType != "" && *result.EntityType != entityType {
 				continue
 			}
 			if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {

pkg/nsx/services/vpc/builder.go

@@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			return nil, nil
 		}
 		// for updating vpc case, use current vpc id, name
-		vpc = nsxVPC
+		*vpc = *nsxVPC
 	} else {
 		// for creating vpc case, fill in vpc properties based on networkconfig
 		vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
@@ -89,3 +89,16 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 
 	return vpc, nil
 }
+
+func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
+	lbs := &model.LBService{}
+	lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
+	lbs.Id = common.String(string(nsObj.GetUID()))
+	lbs.DisplayName = &lbsName
+	// TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag?
+	lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
+	lbs.Size = &lbsSize
+	lbs.ConnectivityPath = &vpcPath
+	lbs.RelaxScaleValidation = relaxScaleValidation
+	return lbs, nil
+}

pkg/nsx/services/vpc/vpc.go

@@ -30,6 +30,7 @@ const (
 	AviSEIngressAllowRuleId    = "avi-se-ingress-allow-rule"
 	VPCAviSEGroupId            = "avi-se-vms"
 	VpcDefaultSecurityPolicyId = "default-layer3-section"
+	VPCKey                     = "/orgs/%s/projects/%s/vpcs/%s"
 	GroupKey                   = "/orgs/%s/projects/%s/vpcs/%s/groups/%s"
 	SecurityPolicyKey          = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s"
 	RuleKey                    = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s"
@@ -41,8 +42,9 @@ var (
 	ResourceTypeVPC = common.ResourceTypeVpc
 	NewConverter    = common.NewConverter
 
-	MarkedForDelete    = true
-	enableAviAllowRule = false
+	MarkedForDelete           = true
+	enableAviAllowRule        = false
+	EnforceRevisionCheckParam = false
 )
 
 type VPCNetworkInfoStore struct {
@@ -568,8 +570,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 		return existingVPC[0], &nc, nil
 	}
 
+	// build NSX LBS
+	var createdLBS *model.LBService
+	// TODO(gran) use switch to enable/disable NSXLB and AVI
+	if enableNSXLB {
+		lbsSize := string(s.NSXConfig.NsxConfig.ServiceSize)
+		vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name)
+		if nc.LbServiceSize != "" {
+			vpcPath = nc.LbServiceSize
+		}
+		var relaxScaleValidation *bool
+		if s.NSXConfig.NsxConfig.RelaxScaleValidaion {
+			relaxScaleValidation = common.Bool(true)
+		}
+		createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation)
+	}
+	// build HAPI request
+	infra, err := s.WrapHierarchyVPC(createdVpc, createdLBS)
+	if err != nil {
+		log.Error(err, "failed to build HAPI request")
+		return nil, nil, err
+	}
+
 	log.Info("creating NSX VPC", "VPC", *createdVpc.Id)
-	err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc)
+	err = s.NSXClient.ProjectInfraClient.Patch(nc.Org, nc.NsxtProject, *infra, &EnforceRevisionCheckParam)
 	err = nsxutil.NSXApiError(err)
 	if err != nil {
 		log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace)
@@ -611,6 +635,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 	}
 
 	s.VpcStore.Add(&newVpc)
+
+	// Check LBS realization
+	if createdLBS != nil {
+		newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id)
+		if err != nil {
+			log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id)
+			return nil, nil, err
+		}
+
+		realizeService := realizestate.InitializeRealizeState(s.Service)
+		if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newLBS.Path, ""); err != nil {
+			log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id)
+			if realizestate.IsRealizeStateError(err) {
+				log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id)
+				// delete the nsx vpc object and re-created in next loop
+				if err := s.DeleteVPC(*newVpc.Path); err != nil {
+					log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id)
+					return nil, nil, err
+				}
+			}
+			return nil, nil, err
+		}
+	}
+
 	return &newVpc, &nc, nil
 }
 
@@ -902,3 +950,17 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo {
 	}
 	return VPCInfoList
 }
+
+func (s *VPCService) GetNSXLBSPath(org, nsxtProject, vpcId, ns string) string {
+	nsObj := &v1.Namespace{}
+	// get ns obj
+	if err := s.Client.Get(ctx, types.NamespacedName{Name: ns}, nsObj); err != nil {
+		log.Error(err, "unable to fetch namespace", "name", ns)
+		return ""
+	}
+	vpcLBS, err := s.NSXClient.VPCLBSClient.Get(org, nsxtProject, vpcId, string(nsObj.GetUID()))
+	if err != nil {
+		return ""
+	}
+	return *vpcLBS.Path
+}

pkg/nsx/services/vpc/wrap.go

@@ -0,0 +1,30 @@
+package vpc
+
+import (
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+func (s *VPCService) WrapHierarchyVPC(vpc *model.Vpc, lbs *model.LBService) (*model.Infra, error) {
+	if lbs != nil {
+		var vpcChildren []*data.StructValue
+		lbsChildren, err := s.WrapLBS(lbs)
+		if err != nil {
+			return nil, err
+		}
+		vpcChildren = append(vpcChildren, lbsChildren...)
+		vpc.Children = vpcChildren
+	}
+	var infraChildren []*data.StructValue
+	vpcChildren, err := s.WrapVPC(vpc)
+	if err != nil {
+		return nil, err
+	}
+	infraChildren = append(infraChildren, vpcChildren...)
+
+	infra, err := s.WrapInfra(infraChildren)
+	if err != nil {
+		return nil, err
+	}
+	return infra, nil
+}