diff --git a/pkg/config/config.go b/pkg/config/config.go index 77b6f33b7..36183e63a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -417,3 +417,10 @@ func (coeConfig *CoeConfig) validate() error { func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error { return nsxConfig.validate(true) } + +func (nsxConfig *NsxConfig) NSXLBEnabled() bool { + if nsxConfig.UseAVILB == false && (nsxConfig.UseNativeLoadBalancer == nil || *nsxConfig.UseNativeLoadBalancer == true) { + return true + } + return false +} diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index 06e62de74..9209fd437 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) return common.ResultRequeue, err } - if !isShared { + if r.Service.NSXConfig.NsxConfig.UseAVILB && !isShared { err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace) if err != nil { state := &v1alpha1.VPCState{ @@ -113,10 +113,11 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // TODO(gran) check if we have NSXLBSubnetInfo // if lb vpc enabled, read avi subnet path and cidr // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back, // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. - if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { + if r.Service.NSXConfig.NsxConfig.UseAVILB && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) if err != nil { log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id) @@ -139,7 +140,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) LoadBalancerIPAddresses: cidr, PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs, } - updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path) + updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(nc.Org, nc.NsxtProject, *createdVpc.Id, obj.Namespace)) } else { if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 076301185..6bee7d312 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ } func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client, - vpcState *v1alpha1.VPCState, ncName string, subnetPath string) { + vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) { setNetworkInfoVPCStatus(c, o, client, vpcState) // ako needs to know the avi subnet path created by nsx - setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath) + setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath) r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated") metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo) } @@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network } } -func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) { +func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) { // read v1alpha1.VPCNetworkConfiguration by ncName nc := &v1alpha1.VPCNetworkConfiguration{} err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc) @@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client createdVPCInfo := &v1alpha1.VPCInfo{ Name: vpcName, AVISESubnetPath: aviSubnetPath, + NSXLBSPath: nsxLBSPath, } // iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it for i, vpc := range nc.Status.VPCs { diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 412b0bd1e..0fd74bd18 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -84,6 +84,7 @@ type Client struct { IPAllocationClient ip_pools.IpAllocationsClient SubnetsClient vpcs.SubnetsClient RealizedStateClient realized_state.RealizedEntitiesClient + VPCLBSClient vpcs.VpcLbsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { SubnetStatusClient: subnetStatusClient, VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, + VPCLBSClient: vpcLBSClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 08d671dff..bca5d1c5e 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -144,6 +144,7 @@ var ( ResourceTypeVpc = "Vpc" ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go new file mode 100644 index 000000000..751e16470 --- /dev/null +++ b/pkg/nsx/services/common/wrap.go @@ -0,0 +1,92 @@ +package common + +import ( + "github.com/openlyinc/pointy" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// WrapInfra TODO(gran) refactor existing code in other package +func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) { + // This is the outermost layer of the hierarchy infra client. + // It doesn't need ID field. + resourceType := ResourceTypeInfra + infraObj := model.Infra{ + Children: children, + ResourceType: &resourceType, + } + return &infraObj, nil +} + +func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) { + resourceType := ResourceTypeOrgRoot + orgRootObj := model.OrgRoot{ + Children: children, + ResourceType: &resourceType, + } + return &orgRootObj, nil +} + +func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeOrg + resourceType := ResourceTypeChildResourceReference + + childOrg := model.ChildResourceReference{ + Id: &org, + ResourceType: resourceType, + TargetType: &targetType, + Children: children, + } + dataValue, errors := NewConverter().ConvertToVapi(childOrg, childOrg.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} + +func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeProject + resourceType := ResourceTypeChildResourceReference + + childProject := model.ChildResourceReference{ + Id: &nsxtProject, + ResourceType: resourceType, + TargetType: &targetType, + Children: children, + } + dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} + +func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) { + vpc.ResourceType = pointy.String(ResourceTypeVpc) + childVpc := model.ChildVpc{ + Id: vpc.Id, + MarkedForDelete: vpc.MarkedForDelete, + ResourceType: "ChildVpc", + Vpc: vpc, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} + +func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) { + lbs.ResourceType = pointy.String(ResourceTypeLBService) + childLBService := model.ChildLBService{ + Id: lbs.Id, + MarkedForDelete: lbs.MarkedForDelete, + ResourceType: "ChildLBService", + LbService: lbs, + } + dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} diff --git a/pkg/nsx/services/realizestate/realize_state.go b/pkg/nsx/services/realizestate/realize_state.go index 8fef3bf8a..1a27350a4 100644 --- a/pkg/nsx/services/realizestate/realize_state.go +++ b/pkg/nsx/services/realizestate/realize_state.go @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte return err } for _, result := range results.Results { - if *result.EntityType != entityType { + if entityType != "" && *result.EntityType != entityType { continue } if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED { diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 5189ac73e..401d52967 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return nil, nil } // for updating vpc case, use current vpc id, name - vpc = nsxVPC + *vpc = *nsxVPC } else { // for creating vpc case, fill in vpc properties based on networkconfig vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster) @@ -89,3 +89,16 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return vpc, nil } + +func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) { + lbs := &model.LBService{} + lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster) + lbs.Id = common.String(string(nsObj.GetUID())) + lbs.DisplayName = &lbsName + // TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag? + lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + lbs.Size = &lbsSize + lbs.ConnectivityPath = &vpcPath + lbs.RelaxScaleValidation = relaxScaleValidation + return lbs, nil +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 415b92773..b7a64a387 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/vmware-tanzu/nsx-operator/pkg/config" "math" "net" "strings" @@ -30,6 +31,7 @@ const ( AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule" VPCAviSEGroupId = "avi-se-vms" VpcDefaultSecurityPolicyId = "default-layer3-section" + VPCKey = "/orgs/%s/projects/%s/vpcs/%s" GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s" SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s" RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s" @@ -41,8 +43,9 @@ var ( ResourceTypeVPC = common.ResourceTypeVpc NewConverter = common.NewConverter - MarkedForDelete = true - enableAviAllowRule = false + MarkedForDelete = true + enableAviAllowRule = false + EnforceRevisionCheckParam = false ) type VPCNetworkInfoStore struct { @@ -568,8 +571,33 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return existingVPC[0], &nc, nil } + // build NSX LBS + var createdLBS *model.LBService + // TODO(gran) use switch to enable/disable NSXLB and AVI + if s.NSXConfig.NsxConfig.NSXLBEnabled() { + lbsSize := s.NSXConfig.NsxConfig.ServiceSize + if lbsSize == "" { + lbsSize = config.LB_SERVICE_SIZE_SMALL + } + if nc.LbServiceSize != "" { + lbsSize = nc.LbServiceSize + } + vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name) + var relaxScaleValidation *bool + if s.NSXConfig.NsxConfig.RelaxScaleValidaion { + relaxScaleValidation = common.Bool(true) + } + createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) + } + // build HAPI request + orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NsxtProject, createdVpc, createdLBS) + if err != nil { + log.Error(err, "failed to build HAPI request") + return nil, nil, err + } + log.Info("creating NSX VPC", "VPC", *createdVpc.Id) - err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc) + err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &EnforceRevisionCheckParam) err = nsxutil.NSXApiError(err) if err != nil { log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace) @@ -611,6 +639,30 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * } s.VpcStore.Add(&newVpc) + + // Check LBS realization + if createdLBS != nil { + newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id) + if err != nil { + log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id) + return nil, nil, err + } + + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newLBS.Path, ""); err != nil { + log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id) + // delete the nsx vpc object and re-created in next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, nil, err + } + } + return nil, nil, err + } + } + return &newVpc, &nc, nil } @@ -902,3 +954,17 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { } return VPCInfoList } + +func (s *VPCService) GetNSXLBSPath(org, nsxtProject, vpcId, ns string) string { + nsObj := &v1.Namespace{} + // get ns obj + if err := s.Client.Get(ctx, types.NamespacedName{Name: ns}, nsObj); err != nil { + log.Error(err, "unable to fetch namespace", "name", ns) + return "" + } + vpcLBS, err := s.NSXClient.VPCLBSClient.Get(org, nsxtProject, vpcId, string(nsObj.GetUID())) + if err != nil { + return "" + } + return *vpcLBS.Path +} diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go new file mode 100644 index 000000000..8b16e771b --- /dev/null +++ b/pkg/nsx/services/vpc/wrap.go @@ -0,0 +1,41 @@ +package vpc + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) { + if lbs != nil { + var vpcChildren []*data.StructValue + childrenLBS, err := s.WrapLBS(lbs) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, childrenLBS...) + vpc.Children = vpcChildren + } + var projectChildren []*data.StructValue + childrenVPC, err := s.WrapVPC(vpc) + if err != nil { + return nil, err + } + projectChildren = append(projectChildren, childrenVPC...) + + var orgChildren []*data.StructValue + childrenProject, err := s.WrapProject(nsxtProject, projectChildren) + if err != nil { + return nil, err + } + orgChildren = append(orgChildren, childrenProject...) + + var orgRootChildren []*data.StructValue + childrenOrg, err := s.WrapOrg(org, orgChildren) + if err != nil { + return nil, err + } + orgRootChildren = append(orgRootChildren, childrenOrg...) + + orgRoot, _ := s.WrapOrgRoot(orgRootChildren) + return orgRoot, nil +}