From ca8f40b08c134e011a4c4269f57ac2dc32cb8437 Mon Sep 17 00:00:00 2001 From: gran Date: Mon, 24 Jun 2024 15:17:34 +0800 Subject: [PATCH] [VPC] Support NSXLB for VPC Signed-off-by: gran --- .../networkinfo/networkinfo_controller.go | 2 +- .../networkinfo/networkinfo_utils.go | 7 +- pkg/nsx/client.go | 3 + pkg/nsx/services/common/types.go | 1 + pkg/nsx/services/common/wrap.go | 53 +++++++++++++++ .../services/realizestate/realize_state.go | 2 +- pkg/nsx/services/vpc/builder.go | 15 +++- pkg/nsx/services/vpc/vpc.go | 68 ++++++++++++++++++- pkg/nsx/services/vpc/wrap.go | 30 ++++++++ 9 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 pkg/nsx/services/common/wrap.go create mode 100644 pkg/nsx/services/vpc/wrap.go diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index 06e62de74..24299e8d3 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) LoadBalancerIPAddresses: cidr, PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs, } - updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path) + updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(nc.Org, nc.NsxtProject, *createdVpc.Id, obj.Namespace)) } else { if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 076301185..6bee7d312 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ } func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client, - vpcState *v1alpha1.VPCState, ncName string, subnetPath string) { + vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) { setNetworkInfoVPCStatus(c, o, client, vpcState) // ako needs to know the avi subnet path created by nsx - setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath) + setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath) r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated") metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo) } @@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network } } -func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) { +func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) { // read v1alpha1.VPCNetworkConfiguration by ncName nc := &v1alpha1.VPCNetworkConfiguration{} err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc) @@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client createdVPCInfo := &v1alpha1.VPCInfo{ Name: vpcName, AVISESubnetPath: aviSubnetPath, + NSXLBSPath: nsxLBSPath, } // iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it for i, vpc := range nc.Status.VPCs { diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 412b0bd1e..0fd74bd18 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -84,6 +84,7 @@ type Client struct { IPAllocationClient ip_pools.IpAllocationsClient SubnetsClient vpcs.SubnetsClient RealizedStateClient realized_state.RealizedEntitiesClient + VPCLBSClient vpcs.VpcLbsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { SubnetStatusClient: subnetStatusClient, VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, + VPCLBSClient: vpcLBSClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 08d671dff..bca5d1c5e 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -144,6 +144,7 @@ var ( ResourceTypeVpc = "Vpc" ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go new file mode 100644 index 000000000..ed1241808 --- /dev/null +++ b/pkg/nsx/services/common/wrap.go @@ -0,0 +1,53 @@ +package common + +import ( + "github.com/openlyinc/pointy" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// WrapInfra TODO(gran) refactor existing code in other package +func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) { + // This is the outermost layer of the hierarchy infra client. + // It doesn't need ID field. + resourceType := ResourceTypeInfra + infraObj := model.Infra{ + Children: children, + ResourceType: &resourceType, + } + return &infraObj, nil +} + +func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) { + var vpcChildren []*data.StructValue + vpc.ResourceType = pointy.String(ResourceTypeVpc) + childVpc := model.ChildVpc{ + Id: vpc.Id, + MarkedForDelete: vpc.MarkedForDelete, + ResourceType: "ChildVpc", + Vpc: vpc, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + vpcChildren = append(vpcChildren, dataValue.(*data.StructValue)) + return vpcChildren, nil +} + +func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) { + var lbServiceChildren []*data.StructValue + lbs.ResourceType = pointy.String(ResourceTypeLBService) + childLBService := model.ChildLBService{ + Id: lbs.Id, + MarkedForDelete: lbs.MarkedForDelete, + ResourceType: "ChildLBService", + LbService: lbs, + } + dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + lbServiceChildren = append(lbServiceChildren, dataValue.(*data.StructValue)) + return lbServiceChildren, nil +} diff --git a/pkg/nsx/services/realizestate/realize_state.go b/pkg/nsx/services/realizestate/realize_state.go index 8fef3bf8a..1a27350a4 100644 --- a/pkg/nsx/services/realizestate/realize_state.go +++ b/pkg/nsx/services/realizestate/realize_state.go @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte return err } for _, result := range results.Results { - if *result.EntityType != entityType { + if entityType != "" && *result.EntityType != entityType { continue } if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED { diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 5189ac73e..401d52967 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return nil, nil } // for updating vpc case, use current vpc id, name - vpc = nsxVPC + *vpc = *nsxVPC } else { // for creating vpc case, fill in vpc properties based on networkconfig vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster) @@ -89,3 +89,16 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return vpc, nil } + +func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) { + lbs := &model.LBService{} + lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster) + lbs.Id = common.String(string(nsObj.GetUID())) + lbs.DisplayName = &lbsName + // TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag? + lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + lbs.Size = &lbsSize + lbs.ConnectivityPath = &vpcPath + lbs.RelaxScaleValidation = relaxScaleValidation + return lbs, nil +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 415b92773..b46b4b52d 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -30,6 +30,7 @@ const ( AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule" VPCAviSEGroupId = "avi-se-vms" VpcDefaultSecurityPolicyId = "default-layer3-section" + VPCKey = "/orgs/%s/projects/%s/vpcs/%s" GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s" SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s" RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s" @@ -41,8 +42,9 @@ var ( ResourceTypeVPC = common.ResourceTypeVpc NewConverter = common.NewConverter - MarkedForDelete = true - enableAviAllowRule = false + MarkedForDelete = true + enableAviAllowRule = false + EnforceRevisionCheckParam = false ) type VPCNetworkInfoStore struct { @@ -568,8 +570,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return existingVPC[0], &nc, nil } + // build NSX LBS + var createdLBS *model.LBService + // TODO(gran) use switch to enable/disable NSXLB and AVI + if enableNSXLB { + lbsSize := string(s.NSXConfig.NsxConfig.ServiceSize) + vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name) + if nc.LbServiceSize != "" { + vpcPath = nc.LbServiceSize + } + var relaxScaleValidation *bool + if s.NSXConfig.NsxConfig.RelaxScaleValidaion { + relaxScaleValidation = common.Bool(true) + } + createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) + } + // build HAPI request + infra, err := s.WrapHierarchyVPC(createdVpc, createdLBS) + if err != nil { + log.Error(err, "failed to build HAPI request") + return nil, nil, err + } + log.Info("creating NSX VPC", "VPC", *createdVpc.Id) - err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc) + err = s.NSXClient.ProjectInfraClient.Patch(nc.Org, nc.NsxtProject, *infra, &EnforceRevisionCheckParam) err = nsxutil.NSXApiError(err) if err != nil { log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace) @@ -611,6 +635,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * } s.VpcStore.Add(&newVpc) + + // Check LBS realization + if createdLBS != nil { + newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id) + if err != nil { + log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id) + return nil, nil, err + } + + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newLBS.Path, ""); err != nil { + log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id) + // delete the nsx vpc object and re-created in next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, nil, err + } + } + return nil, nil, err + } + } + return &newVpc, &nc, nil } @@ -902,3 +950,17 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { } return VPCInfoList } + +func (s *VPCService) GetNSXLBSPath(org, nsxtProject, vpcId, ns string) string { + nsObj := &v1.Namespace{} + // get ns obj + if err := s.Client.Get(ctx, types.NamespacedName{Name: ns}, nsObj); err != nil { + log.Error(err, "unable to fetch namespace", "name", ns) + return "" + } + vpcLBS, err := s.NSXClient.VPCLBSClient.Get(org, nsxtProject, vpcId, string(nsObj.GetUID())) + if err != nil { + return "" + } + return *vpcLBS.Path +} diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go new file mode 100644 index 000000000..59db4a271 --- /dev/null +++ b/pkg/nsx/services/vpc/wrap.go @@ -0,0 +1,30 @@ +package vpc + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func (s *VPCService) WrapHierarchyVPC(vpc *model.Vpc, lbs *model.LBService) (*model.Infra, error) { + if lbs != nil { + var vpcChildren []*data.StructValue + lbsChildren, err := s.WrapLBS(lbs) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, lbsChildren...) + vpc.Children = vpcChildren + } + var infraChildren []*data.StructValue + vpcChildren, err := s.WrapVPC(vpc) + if err != nil { + return nil, err + } + infraChildren = append(infraChildren, vpcChildren...) + + infra, err := s.WrapInfra(infraChildren) + if err != nil { + return nil, err + } + return infra, nil +}