Check loadbalancer type (#663)

if use_avi_lb is true in the config, nsx-operator will also check
1. if alb endpoint existed
2. no nsx lbs resources found
if all conditions met, nsx-operator will return 'avi', else return
'nsx-lb'

Test Done:
1. set use_avi_lb true in config, get avi endpoint, no nsx lbs , return "avi"
2. set use_avi_lb true in config, no avi endpoint, return "nsx-lb"
3. set use_avi_lb false in config, return "nsx-lb"

pkg/config/config.go

@@ -411,13 +411,6 @@ func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error {
 	return nsxConfig.validate(true)
 }
 
-func (nsxConfig *NsxConfig) NSXLBEnabled() bool {
-	if nsxConfig.UseAVILoadBalancer == false && (nsxConfig.UseNSXLoadBalancer == nil || *nsxConfig.UseNSXLoadBalancer == true) {
-		return true
-	}
-	return false
-}
-
 func (nsxConfig *NsxConfig) GetNSXLBSize() string {
 	lbsSize := nsxConfig.NSXLBSize
 	if lbsSize == "" {

pkg/config/config_test.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"testing"
 
-	"github.com/openlyinc/pointy"
 	"github.com/stretchr/testify/assert"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
 )
@@ -179,56 +178,6 @@ func TestNSXOperatorConfig_GetCACert(t *testing.T) {
 	}
 }
 
-func TestNsxConfig_NSXLBEnabled(t *testing.T) {
-	type fields struct {
-		UseAVILB              bool
-		UseNativeLoadBalancer *bool
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   bool
-	}{{
-		name: "avilb",
-		fields: fields{
-			UseAVILB:              true,
-			UseNativeLoadBalancer: nil,
-		},
-		want: false,
-	}, {
-		name: "nsxlbnil",
-		fields: fields{
-			UseAVILB:              false,
-			UseNativeLoadBalancer: nil,
-		},
-		want: true,
-	}, {
-		name: "nsxlbtrue",
-		fields: fields{
-			UseAVILB:              false,
-			UseNativeLoadBalancer: pointy.Bool(true),
-		},
-		want: true,
-	}, {
-		name: "nsxlbfalse",
-		fields: fields{
-			UseAVILB:              false,
-			UseNativeLoadBalancer: pointy.Bool(false),
-		},
-		want: false,
-	},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			nsxConfig := &NsxConfig{
-				UseAVILoadBalancer: tt.fields.UseAVILB,
-				UseNSXLoadBalancer: tt.fields.UseNativeLoadBalancer,
-			}
-			assert.Equalf(t, tt.want, nsxConfig.NSXLBEnabled(), "NSXLBEnabled()")
-		})
-	}
-}
-
 func TestNsxConfig_GetServiceSize(t *testing.T) {
 	type fields struct {
 		ServiceSize string

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -114,7 +114,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		// if lb vpc enabled, read avi subnet path and cidr
 		// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
 		// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
-		if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
+		if !r.Service.NSXLBEnabled() && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
 			path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
 			if err != nil {
 				log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)

pkg/nsx/services/realizestate/realize_state.go

@@ -4,7 +4,6 @@
 package realizestate
 
 import (
-	"errors"
 	"fmt"
 
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
@@ -35,6 +34,7 @@ func IsRealizeStateError(err error) bool {
 // CheckRealizeState allows the caller to check realize status of entityType with retries.
 // backoff defines the maximum retries and the wait interval between two retries.
 func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, intentPath, entityType string) error {
+	// TODO， ask NSX if there were multiple realize states could we check only the latest one?
 	vpcInfo, err := common.ParseVPCResourcePath(intentPath)
 	if err != nil {
 		return err
@@ -55,7 +55,6 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
 			if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {
 				return nil
 			}
-			return errors.New(*result.State)
 		}
 		return fmt.Errorf("%s not realized", entityType)
 	})

pkg/nsx/services/vpc/vpc.go

@@ -12,6 +12,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/retry"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
@@ -30,14 +31,18 @@ const (
 	GroupKey                   = "/orgs/%s/projects/%s/vpcs/%s/groups/%s"
 	SecurityPolicyKey          = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s"
 	RuleKey                    = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s"
+	albEndpointPath            = "policy/api/v1/infra/sites/default/enforcement-points/alb-endpoint"
+	LBProviderNSX              = "nsx-lb"
+	LBProviderAVI              = "avi"
 )
 
 var (
-	log             = &logger.Log
-	ctx             = context.Background()
-	ResourceTypeVPC = common.ResourceTypeVpc
-	NewConverter    = common.NewConverter
-
+	log                       = &logger.Log
+	ctx                       = context.Background()
+	ResourceTypeVPC           = common.ResourceTypeVpc
+	NewConverter              = common.NewConverter
+	lbProvider                = ""
+	lbProviderMutex           = &sync.Mutex{}
 	MarkedForDelete           = true
 	enableAviAllowRule        = false
 	EnforceRevisionCheckParam = false
@@ -568,7 +573,7 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 		nsxVPC = nil
 	}
 
-	createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC, s.NSXConfig.NsxConfig.UseAVILoadBalancer)
+	createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC, !s.NSXLBEnabled())
 	if err != nil {
 		log.Error(err, "failed to build NSX VPC object")
 		return nil, nil, err
@@ -582,7 +587,7 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
 
 	// build NSX LBS
 	var createdLBS *model.LBService
-	if s.NSXConfig.NsxConfig.NSXLBEnabled() {
+	if s.NSXLBEnabled() {
 		lbsSize := s.NSXConfig.NsxConfig.GetNSXLBSize()
 		vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NSXProject, nc.Name)
 		var relaxScaleValidation *bool
@@ -681,8 +686,10 @@ func (s *VPCService) Cleanup(ctx context.Context) error {
 			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
 		default:
 			// first clean avi subnet ports, or else vpc delete will fail
-			if err := CleanAviSubnetPorts(ctx, s.NSXClient.Cluster, *vpc.Path); err != nil {
-				return err
+			if !s.NSXLBEnabled() {
+				if err := CleanAviSubnetPorts(ctx, s.NSXClient.Cluster, *vpc.Path); err != nil {
+					return err
+				}
 			}
 			if err := s.DeleteVPC(*vpc.Path); err != nil {
 				return err
@@ -728,3 +735,49 @@ func (s *VPCService) GetNSXLBSPath(lbsId string) string {
 	}
 	return *vpcLBS.Path
 }
+
+func GetAlbEndpoint(cluster *nsx.Cluster) error {
+	_, err := cluster.HttpGet(albEndpointPath)
+	return err
+}
+
+func (vpcService *VPCService) NSXLBEnabled() bool {
+	lbProviderMutex.Lock()
+	defer lbProviderMutex.Unlock()
+
+	if lbProvider == "" {
+		lbProvider = vpcService.getLBProvider()
+	}
+	return lbProvider == LBProviderNSX
+}
+
+func (vpcService *VPCService) getLBProvider() string {
+	// if no Alb endpoint found, return nsx-lb
+	// if found, and nsx lbs found, return nsx-lb
+	// else return avi
+	if !vpcService.Service.NSXConfig.UseAVILoadBalancer {
+		return LBProviderNSX
+	}
+	albEndpointFound := false
+	if err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
+		if err == nil {
+			return false
+		}
+		if errors.Is(err, nsxutil.HttpCommonError) {
+			return true
+		} else {
+			return false
+		}
+	}, func() error {
+		return GetAlbEndpoint(vpcService.Service.NSXClient.Cluster)
+	}); err == nil {
+		albEndpointFound = true
+	}
+	if !albEndpointFound {
+		return LBProviderNSX
+	}
+	if len(vpcService.LbsStore.List()) > 0 {
+		return LBProviderNSX
+	}
+	return LBProviderAVI
+}