[VPC] Support NSXLB for VPC

.golangci.yml

@@ -32,4 +32,10 @@ linters:
     - gosec
     - goimports
     - vet
-    - revive
+    - revive
+
+issues:
+  exclude-rules:
+    - linters:
+        - staticcheck
+      text: "SA1019: lbs.RelaxScaleValidation"

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
 			return common.ResultRequeue, err
 		}
-		if !isShared {
+		if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && !isShared {
 			err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
 			if err != nil {
 				state := &v1alpha1.VPCState{
@@ -116,7 +116,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		// if lb vpc enabled, read avi subnet path and cidr
 		// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
 		// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
-		if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
+		if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
 			path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
 			if err != nil {
 				log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
@@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			LoadBalancerIPAddresses: cidr,
 			PrivateIPv4CIDRs:        nc.PrivateIPv4CIDRs,
 		}
-		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
+		updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id))
 	} else {
 		if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
 			metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
 }
 
 func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
-	vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
+	vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
 	setNetworkInfoVPCStatus(c, o, client, vpcState)
 	// ako needs to know the avi subnet path created by nsx
-	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
+	setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
 	r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
 }
@@ -59,16 +59,17 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
 	}
 }
 
-func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
+func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
 	// read v1alpha1.VPCNetworkConfiguration by ncName
 	nc := &v1alpha1.VPCNetworkConfiguration{}
 	err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
 	if err != nil {
 		log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName)
 	}
 	createdVPCInfo := &v1alpha1.VPCInfo{
-		Name:            vpcName,
-		AVISESubnetPath: aviSubnetPath,
+		Name:                vpcName,
+		AVISESubnetPath:     aviSubnetPath,
+		NSXLoadBalancerPath: nsxLBSPath,
 	}
 	// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
 	for i, vpc := range nc.Status.VPCs {

pkg/nsx/client.go

@@ -85,6 +85,7 @@ type Client struct {
 	SubnetsClient             vpcs.SubnetsClient
 	RealizedStateClient       realized_state.RealizedEntitiesClient
 	IPAddressAllocationClient vpcs.IpAddressAllocationsClient
+	VPCLBSClient              vpcs.VpcLbsClient
 
 	NSXChecker    NSXHealthChecker
 	NSXVerChecker NSXVersionChecker
@@ -165,6 +166,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
 	ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster))
+	vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
 
 	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
 	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -206,6 +208,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		SubnetStatusClient: subnetStatusClient,
 		VPCSecurityClient:  vpcSecurityClient,
 		VPCRuleClient:      vpcRuleClient,
+		VPCLBSClient:       vpcLBSClient,
 
 		NSXChecker:                *nsxChecker,
 		NSXVerChecker:             *nsxVersionChecker,

pkg/nsx/services/common/types.go

@@ -28,6 +28,7 @@ const (
 	TagScopeNCPVIFProjectUID           string = "ncp/vif_project_uid"
 	TagScopeNCPPod                     string = "ncp/pod"
 	TagScopeNCPVNETInterface           string = "ncp/vnet_interface"
+	TagScopeCreatedFor                 string = "nsx-op/created_for"
 	TagScopeVersion                    string = "nsx-op/version"
 	TagScopeCluster                    string = "nsx-op/cluster"
 	TagScopeNamespace                  string = "nsx-op/namespace"
@@ -74,6 +75,7 @@ const (
 	TagValueGroupSource                string = "source"
 	TagValueGroupDestination           string = "destination"
 	TagValueGroupAvi                   string = "avi"
+	TagValueSLB                        string = "SLB"
 	AnnotationVPCNetworkConfig         string = "nsx.vmware.com/vpc_network_config"
 	AnnotationSharedVPCNamespace       string = "nsx.vmware.com/shared_vpc_namespace"
 	AnnotationDefaultNetworkConfig     string = "nsx.vmware.com/default"
@@ -147,6 +149,7 @@ var (
 	ResourceTypeVpc                    = "Vpc"
 	ResourceTypeSubnetPort             = "VpcSubnetPort"
 	ResourceTypeVirtualMachine         = "VirtualMachine"
+	ResourceTypeLBService              = "LBService"
 	ResourceTypeShare                  = "Share"
 	ResourceTypeSharedResource         = "SharedResource"
 	ResourceTypeChildSharedResource    = "ChildSharedResource"

pkg/nsx/services/common/wrap.go

@@ -0,0 +1,84 @@
+package common
+
+import (
+	"github.com/openlyinc/pointy"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+)
+
+// WrapInfra TODO(gran) refactor existing code in other package
+func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
+	// This is the outermost layer of the hierarchy infra client.
+	// It doesn't need ID field.
+	resourceType := ResourceTypeInfra
+	infraObj := model.Infra{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &infraObj, nil
+}
+
+func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
+	resourceType := ResourceTypeOrgRoot
+	orgRootObj := model.OrgRoot{
+		Children:     children,
+		ResourceType: &resourceType,
+	}
+	return &orgRootObj, nil
+}
+
+func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeOrg
+	return wrapChildResourceReference(targetType, org, children)
+}
+
+func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
+	targetType := ResourceTypeProject
+	return wrapChildResourceReference(targetType, nsxtProject, children)
+}
+
+func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) {
+	resourceType := ResourceTypeChildResourceReference
+	childProject := model.ChildResourceReference{
+		Id:           &id,
+		ResourceType: resourceType,
+		TargetType:   &targetType,
+		Children:     children,
+	}
+	dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
+	if len(errors) > 0 {
+		return nil, errors[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+
+}
+
+func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
+	vpc.ResourceType = pointy.String(ResourceTypeVpc)
+	childVpc := model.ChildVpc{
+		Id:              vpc.Id,
+		MarkedForDelete: vpc.MarkedForDelete,
+		ResourceType:    "ChildVpc",
+		Vpc:             vpc,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}
+
+func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
+	lbs.ResourceType = pointy.String(ResourceTypeLBService)
+	childLBService := model.ChildLBService{
+		Id:              lbs.Id,
+		MarkedForDelete: lbs.MarkedForDelete,
+		ResourceType:    "ChildLBService",
+		LbService:       lbs,
+	}
+	dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
+	return []*data.StructValue{dataValue.(*data.StructValue)}, nil
+}

pkg/nsx/services/realizestate/realize_state.go

@@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
 			return err
 		}
 		for _, result := range results.Results {
-			if *result.EntityType != entityType {
+			if entityType != "" && result.EntityType != nil && *result.EntityType != entityType {
 				continue
 			}
 			if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {

pkg/nsx/services/vpc/builder.go

@@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace,
 }
 
 func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string,
-	nsxVPC *model.Vpc) (*model.Vpc,
+	nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc,
 	error) {
 	vpc := &model.Vpc{}
 	if nsxVPC != nil {
@@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			return nil, nil
 		}
 		// for updating vpc case, use current vpc id, name
-		vpc = nsxVPC
+		*vpc = *nsxVPC
 	} else {
 		// for creating vpc case, fill in vpc properties based on networkconfig
 		vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
@@ -76,7 +76,10 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			},
 		}
 		vpc.SiteInfos = siteInfos
-		vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
+		if useAVILB {
+			loadBalancerVPCEndpointEnabled := true
+			vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled}
+		}
 		vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
 	}
 
@@ -89,3 +92,21 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 
 	return vpc, nil
 }
+
+func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
+	lbs := &model.LBService{}
+	lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
+	// Use VPC id for auto-created LBS id
+	lbs.Id = common.String(string(nsObj.GetUID()))
+	lbs.DisplayName = &lbsName
+	lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
+	// "created_for" is required by NCP, and "lb_t1_link_ip" is not needed for VPC
+	lbs.Tags = append(lbs.Tags, model.Tag{
+		Scope: common.String(common.TagScopeCreatedFor),
+		Tag:   common.String(common.TagValueSLB),
+	})
+	lbs.Size = &lbsSize
+	lbs.ConnectivityPath = &vpcPath
+	lbs.RelaxScaleValidation = relaxScaleValidation
+	return lbs, nil
+}

pkg/nsx/services/vpc/builder_test.go

@@ -0,0 +1,79 @@
+package vpc
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+)
+
+func Test_buildNSXLBS(t *testing.T) {
+	type args struct {
+		obj                  *v1alpha1.NetworkInfo
+		nsObj                *v1.Namespace
+		cluster              string
+		lbsSize              string
+		vpcPath              string
+		relaxScaleValidation *bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *model.LBService
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "1",
+			args: args{
+				obj: &v1alpha1.NetworkInfo{
+					ObjectMeta: metav1.ObjectMeta{Namespace: "ns1"},
+					VPCs:       nil,
+				},
+				nsObj: &v1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{Name: "ns1", UID: "nsuid1"},
+				},
+				cluster:              "cluster1",
+				lbsSize:              model.LBService_SIZE_SMALL,
+				vpcPath:              "/vpc1",
+				relaxScaleValidation: nil,
+			},
+			want: &model.LBService{
+				Id:          common.String("nsuid1"),
+				DisplayName: common.String("vpc-cluster1--ns1"),
+				Tags: []model.Tag{
+					{
+						Scope: common.String(common.TagScopeCluster),
+						Tag:   common.String("cluster1"),
+					},
+					{
+						Scope: common.String(common.TagScopeVersion),
+						Tag:   common.String(strings.Join(common.TagValueVersion, ".")),
+					},
+					{Scope: common.String(common.TagScopeNamespace), Tag: common.String("ns1")},
+					{Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String("nsuid1")},
+					{Scope: common.String(common.TagScopeCreatedFor), Tag: common.String(common.TagValueSLB)},
+				},
+				Size:                 common.String(model.LBService_SIZE_SMALL),
+				ConnectivityPath:     common.String("/vpc1"),
+				RelaxScaleValidation: nil,
+			},
+			wantErr: assert.NoError,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildNSXLBS(tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
+			if !tt.wantErr(t, err, fmt.Sprintf("buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)) {
+				return
+			}
+			assert.Equalf(t, tt.want, got, "buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
+		})
+	}
+}