Skip to content

Commit

Permalink
[VPC] Support NSXLB for VPC
Browse files Browse the repository at this point in the history
Signed-off-by: gran <gran@vmware.com>
  • Loading branch information
gran-vmv committed Jul 4, 2024
1 parent 6522fff commit f9337b2
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 11 deletions.
7 changes: 7 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -417,3 +417,10 @@ func (coeConfig *CoeConfig) validate() error {
func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error {
return nsxConfig.validate(true)
}

func (nsxConfig *NsxConfig) NSXLBEnabled() bool {
if nsxConfig.UseAVILB == false && (nsxConfig.UseNativeLoadBalancer == nil || *nsxConfig.UseNativeLoadBalancer == true) {
return true
}
return false
}
7 changes: 4 additions & 3 deletions pkg/controllers/networkinfo/networkinfo_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
return common.ResultRequeue, err
}
if !isShared {
if r.Service.NSXConfig.NsxConfig.UseAVILB && !isShared {
err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
if err != nil {
state := &v1alpha1.VPCState{
Expand Down Expand Up @@ -113,10 +113,11 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
}

// TODO(gran) check if we have NSXLBSubnetInfo
// if lb vpc enabled, read avi subnet path and cidr
// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
if r.Service.NSXConfig.NsxConfig.UseAVILB && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
if err != nil {
log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
Expand All @@ -139,7 +140,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
LoadBalancerIPAddresses: cidr,
PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs,
}
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(nc.Org, nc.NsxtProject, *createdVpc.Id, obj.Namespace))
} else {
if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)
Expand Down
7 changes: 4 additions & 3 deletions pkg/controllers/networkinfo/networkinfo_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
}

func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
setNetworkInfoVPCStatus(c, o, client, vpcState)
// ako needs to know the avi subnet path created by nsx
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
}
Expand All @@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
}
}

func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
// read v1alpha1.VPCNetworkConfiguration by ncName
nc := &v1alpha1.VPCNetworkConfiguration{}
err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
Expand All @@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client
createdVPCInfo := &v1alpha1.VPCInfo{
Name: vpcName,
AVISESubnetPath: aviSubnetPath,
NSXLBSPath: nsxLBSPath,
}
// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
for i, vpc := range nc.Status.VPCs {
Expand Down
3 changes: 3 additions & 0 deletions pkg/nsx/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ type Client struct {
IPAllocationClient ip_pools.IpAllocationsClient
SubnetsClient vpcs.SubnetsClient
RealizedStateClient realized_state.RealizedEntitiesClient
VPCLBSClient vpcs.VpcLbsClient

NSXChecker NSXHealthChecker
NSXVerChecker NSXVersionChecker
Expand Down Expand Up @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster))
subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))

vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
Expand Down Expand Up @@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
SubnetStatusClient: subnetStatusClient,
VPCSecurityClient: vpcSecurityClient,
VPCRuleClient: vpcRuleClient,
VPCLBSClient: vpcLBSClient,

NSXChecker: *nsxChecker,
NSXVerChecker: *nsxVersionChecker,
Expand Down
1 change: 1 addition & 0 deletions pkg/nsx/services/common/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ var (
ResourceTypeVpc = "Vpc"
ResourceTypeSubnetPort = "VpcSubnetPort"
ResourceTypeVirtualMachine = "VirtualMachine"
ResourceTypeLBService = "LBService"
ResourceTypeShare = "Share"
ResourceTypeSharedResource = "SharedResource"
ResourceTypeChildSharedResource = "ChildSharedResource"
Expand Down
92 changes: 92 additions & 0 deletions pkg/nsx/services/common/wrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package common

import (
"github.com/openlyinc/pointy"
"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
)

// WrapInfra TODO(gran) refactor existing code in other package
func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
// This is the outermost layer of the hierarchy infra client.
// It doesn't need ID field.
resourceType := ResourceTypeInfra
infraObj := model.Infra{
Children: children,
ResourceType: &resourceType,
}
return &infraObj, nil
}

func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
resourceType := ResourceTypeOrgRoot
orgRootObj := model.OrgRoot{
Children: children,
ResourceType: &resourceType,
}
return &orgRootObj, nil
}

func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
targetType := ResourceTypeOrg
resourceType := ResourceTypeChildResourceReference

childOrg := model.ChildResourceReference{
Id: &org,
ResourceType: resourceType,
TargetType: &targetType,
Children: children,
}
dataValue, errors := NewConverter().ConvertToVapi(childOrg, childOrg.GetType__())
if len(errors) > 0 {
return nil, errors[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}

func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
targetType := ResourceTypeProject
resourceType := ResourceTypeChildResourceReference

childProject := model.ChildResourceReference{
Id: &nsxtProject,
ResourceType: resourceType,
TargetType: &targetType,
Children: children,
}
dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
if len(errors) > 0 {
return nil, errors[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}

func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
vpc.ResourceType = pointy.String(ResourceTypeVpc)
childVpc := model.ChildVpc{
Id: vpc.Id,
MarkedForDelete: vpc.MarkedForDelete,
ResourceType: "ChildVpc",
Vpc: vpc,
}
dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
if len(errs) > 0 {
return nil, errs[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}

func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
lbs.ResourceType = pointy.String(ResourceTypeLBService)
childLBService := model.ChildLBService{
Id: lbs.Id,
MarkedForDelete: lbs.MarkedForDelete,
ResourceType: "ChildLBService",
LbService: lbs,
}
dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
if len(errs) > 0 {
return nil, errs[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}
2 changes: 1 addition & 1 deletion pkg/nsx/services/realizestate/realize_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
return err
}
for _, result := range results.Results {
if *result.EntityType != entityType {
if entityType != "" && *result.EntityType != entityType {
continue
}
if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {
Expand Down
15 changes: 14 additions & 1 deletion pkg/nsx/services/vpc/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
return nil, nil
}
// for updating vpc case, use current vpc id, name
vpc = nsxVPC
*vpc = *nsxVPC
} else {
// for creating vpc case, fill in vpc properties based on networkconfig
vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
Expand Down Expand Up @@ -89,3 +89,16 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe

return vpc, nil
}

func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
lbs := &model.LBService{}
lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
lbs.Id = common.String(string(nsObj.GetUID()))
lbs.DisplayName = &lbsName
// TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag?
lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
lbs.Size = &lbsSize
lbs.ConnectivityPath = &vpcPath
lbs.RelaxScaleValidation = relaxScaleValidation
return lbs, nil
}
72 changes: 69 additions & 3 deletions pkg/nsx/services/vpc/vpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/vmware-tanzu/nsx-operator/pkg/config"
"math"
"net"
"strings"
Expand All @@ -30,6 +31,7 @@ const (
AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule"
VPCAviSEGroupId = "avi-se-vms"
VpcDefaultSecurityPolicyId = "default-layer3-section"
VPCKey = "/orgs/%s/projects/%s/vpcs/%s"
GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s"
SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s"
RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s"
Expand All @@ -41,8 +43,9 @@ var (
ResourceTypeVPC = common.ResourceTypeVpc
NewConverter = common.NewConverter

MarkedForDelete = true
enableAviAllowRule = false
MarkedForDelete = true
enableAviAllowRule = false
EnforceRevisionCheckParam = false
)

type VPCNetworkInfoStore struct {
Expand Down Expand Up @@ -568,8 +571,33 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
return existingVPC[0], &nc, nil
}

// build NSX LBS
var createdLBS *model.LBService
// TODO(gran) use switch to enable/disable NSXLB and AVI
if s.NSXConfig.NsxConfig.NSXLBEnabled() {
lbsSize := s.NSXConfig.NsxConfig.ServiceSize
if lbsSize == "" {
lbsSize = config.LB_SERVICE_SIZE_SMALL
}
if nc.LbServiceSize != "" {

Check failure on line 582 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)

Check failure on line 582 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)

Check failure on line 582 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)

Check failure on line 582 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)
lbsSize = nc.LbServiceSize

Check failure on line 583 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)) (typecheck)

Check failure on line 583 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)) (typecheck)

Check failure on line 583 in pkg/nsx/services/vpc/vpc.go

View workflow job for this annotation

GitHub Actions / build

nc.LbServiceSize undefined (type "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common".VPCNetworkConfigInfo has no field or method LbServiceSize)) (typecheck)
}
vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name)
var relaxScaleValidation *bool
if s.NSXConfig.NsxConfig.RelaxScaleValidaion {
relaxScaleValidation = common.Bool(true)
}
createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation)
}
// build HAPI request
orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NsxtProject, createdVpc, createdLBS)
if err != nil {
log.Error(err, "failed to build HAPI request")
return nil, nil, err
}

log.Info("creating NSX VPC", "VPC", *createdVpc.Id)
err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc)
err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &EnforceRevisionCheckParam)
err = nsxutil.NSXApiError(err)
if err != nil {
log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace)
Expand Down Expand Up @@ -611,6 +639,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, *
}

s.VpcStore.Add(&newVpc)

// Check LBS realization
if createdLBS != nil {
newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id)
if err != nil {
log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id)
return nil, nil, err
}

This comment has been minimized.

Copy link
@TaoZou1

TaoZou1 Jul 5, 2024

Contributor

not using a lbs_store to save the lbs item to check it later? after nsx-operator restart, how does it update the networkconfiguration?

realizeService := realizestate.InitializeRealizeState(s.Service)
if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newLBS.Path, ""); err != nil {
log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id)
if realizestate.IsRealizeStateError(err) {
log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id)
// delete the nsx vpc object and re-created in next loop
if err := s.DeleteVPC(*newVpc.Path); err != nil {
log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id)
return nil, nil, err
}
}
return nil, nil, err
}
}

return &newVpc, &nc, nil
}

Expand Down Expand Up @@ -902,3 +954,17 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo {
}
return VPCInfoList
}

func (s *VPCService) GetNSXLBSPath(org, nsxtProject, vpcId, ns string) string {
nsObj := &v1.Namespace{}
// get ns obj
if err := s.Client.Get(ctx, types.NamespacedName{Name: ns}, nsObj); err != nil {
log.Error(err, "unable to fetch namespace", "name", ns)
return ""
}
vpcLBS, err := s.NSXClient.VPCLBSClient.Get(org, nsxtProject, vpcId, string(nsObj.GetUID()))
if err != nil {
return ""
}
return *vpcLBS.Path
}
41 changes: 41 additions & 0 deletions pkg/nsx/services/vpc/wrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package vpc

import (
"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
)

func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) {
if lbs != nil {
var vpcChildren []*data.StructValue
childrenLBS, err := s.WrapLBS(lbs)
if err != nil {
return nil, err
}
vpcChildren = append(vpcChildren, childrenLBS...)
vpc.Children = vpcChildren
}
var projectChildren []*data.StructValue
childrenVPC, err := s.WrapVPC(vpc)
if err != nil {
return nil, err
}
projectChildren = append(projectChildren, childrenVPC...)

var orgChildren []*data.StructValue
childrenProject, err := s.WrapProject(nsxtProject, projectChildren)
if err != nil {
return nil, err
}
orgChildren = append(orgChildren, childrenProject...)

var orgRootChildren []*data.StructValue
childrenOrg, err := s.WrapOrg(org, orgChildren)
if err != nil {
return nil, err
}
orgRootChildren = append(orgRootChildren, childrenOrg...)

orgRoot, _ := s.WrapOrgRoot(orgRootChildren)
return orgRoot, nil
}

0 comments on commit f9337b2

Please sign in to comment.