diff --git a/.golangci.yml b/.golangci.yml index eec0fea19..c8d26644d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,4 +32,10 @@ linters: - gosec - goimports - vet - - revive \ No newline at end of file + - revive + +issues: + exclude-rules: + - linters: + - staticcheck + text: "SA1019: lbs.RelaxScaleValidation" \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go index 77b6f33b7..36183e63a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -417,3 +417,10 @@ func (coeConfig *CoeConfig) validate() error { func (nsxConfig *NsxConfig) ValidateConfigFromCmd() error { return nsxConfig.validate(true) } + +func (nsxConfig *NsxConfig) NSXLBEnabled() bool { + if nsxConfig.UseAVILB == false && (nsxConfig.UseNativeLoadBalancer == nil || *nsxConfig.UseNativeLoadBalancer == true) { + return true + } + return false +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 6ca245042..5fbacc21f 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -9,6 +9,7 @@ import ( "os" "testing" + "github.com/openlyinc/pointy" "github.com/stretchr/testify/assert" ) @@ -176,3 +177,53 @@ func TestNSXOperatorConfig_GetCACert(t *testing.T) { }) } } + +func TestNsxConfig_NSXLBEnabled(t *testing.T) { + type fields struct { + UseAVILB bool + UseNativeLoadBalancer *bool + } + tests := []struct { + name string + fields fields + want bool + }{{ + name: "avilb", + fields: fields{ + UseAVILB: true, + UseNativeLoadBalancer: nil, + }, + want: false, + }, { + name: "nsxlbnil", + fields: fields{ + UseAVILB: false, + UseNativeLoadBalancer: nil, + }, + want: true, + }, { + name: "nsxlbtrue", + fields: fields{ + UseAVILB: false, + UseNativeLoadBalancer: pointy.Bool(true), + }, + want: true, + }, { + name: "nsxlbfalse", + fields: fields{ + UseAVILB: false, + UseNativeLoadBalancer: pointy.Bool(false), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nsxConfig := &NsxConfig{ + UseAVILB: tt.fields.UseAVILB, + UseNativeLoadBalancer: tt.fields.UseNativeLoadBalancer, + } + assert.Equalf(t, tt.want, nsxConfig.NSXLBEnabled(), "NSXLBEnabled()") + }) + } +} diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index 06e62de74..3faa5a375 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) return common.ResultRequeue, err } - if !isShared { + if r.Service.NSXConfig.NsxConfig.UseAVILB && !isShared { err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace) if err != nil { state := &v1alpha1.VPCState{ @@ -113,10 +113,11 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // TODO(gran) check if we need to query NSXLBSubnetInfo // if lb vpc enabled, read avi subnet path and cidr // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back, // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. - if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { + if r.Service.NSXConfig.NsxConfig.UseAVILB && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) if err != nil { log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id) @@ -139,7 +140,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) LoadBalancerIPAddresses: cidr, PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs, } - updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path) + updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id)) } else { if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 076301185..6bee7d312 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ } func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client, - vpcState *v1alpha1.VPCState, ncName string, subnetPath string) { + vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) { setNetworkInfoVPCStatus(c, o, client, vpcState) // ako needs to know the avi subnet path created by nsx - setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath) + setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath) r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated") metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo) } @@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network } } -func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) { +func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) { // read v1alpha1.VPCNetworkConfiguration by ncName nc := &v1alpha1.VPCNetworkConfiguration{} err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc) @@ -69,6 +69,7 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client createdVPCInfo := &v1alpha1.VPCInfo{ Name: vpcName, AVISESubnetPath: aviSubnetPath, + NSXLBSPath: nsxLBSPath, } // iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it for i, vpc := range nc.Status.VPCs { diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 412b0bd1e..0fd74bd18 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -84,6 +84,7 @@ type Client struct { IPAllocationClient ip_pools.IpAllocationsClient SubnetsClient vpcs.SubnetsClient RealizedStateClient realized_state.RealizedEntitiesClient + VPCLBSClient vpcs.VpcLbsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -163,6 +164,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) + vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -204,6 +206,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { SubnetStatusClient: subnetStatusClient, VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, + VPCLBSClient: vpcLBSClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 08d671dff..bca5d1c5e 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -144,6 +144,7 @@ var ( ResourceTypeVpc = "Vpc" ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go new file mode 100644 index 000000000..a2c96d3a0 --- /dev/null +++ b/pkg/nsx/services/common/wrap.go @@ -0,0 +1,84 @@ +package common + +import ( + "github.com/openlyinc/pointy" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// WrapInfra TODO(gran) refactor existing code in other package +func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) { + // This is the outermost layer of the hierarchy infra client. + // It doesn't need ID field. + resourceType := ResourceTypeInfra + infraObj := model.Infra{ + Children: children, + ResourceType: &resourceType, + } + return &infraObj, nil +} + +func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) { + resourceType := ResourceTypeOrgRoot + orgRootObj := model.OrgRoot{ + Children: children, + ResourceType: &resourceType, + } + return &orgRootObj, nil +} + +func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeOrg + return wrapChildResourceReference(targetType, org, children) +} + +func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeProject + return wrapChildResourceReference(targetType, nsxtProject, children) +} + +func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) { + resourceType := ResourceTypeChildResourceReference + childProject := model.ChildResourceReference{ + Id: &id, + ResourceType: resourceType, + TargetType: &targetType, + Children: children, + } + dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil + +} + +func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) { + vpc.ResourceType = pointy.String(ResourceTypeVpc) + childVpc := model.ChildVpc{ + Id: vpc.Id, + MarkedForDelete: vpc.MarkedForDelete, + ResourceType: "ChildVpc", + Vpc: vpc, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} + +func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) { + lbs.ResourceType = pointy.String(ResourceTypeLBService) + childLBService := model.ChildLBService{ + Id: lbs.Id, + MarkedForDelete: lbs.MarkedForDelete, + ResourceType: "ChildLBService", + LbService: lbs, + } + dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} diff --git a/pkg/nsx/services/realizestate/realize_state.go b/pkg/nsx/services/realizestate/realize_state.go index 8fef3bf8a..1a27350a4 100644 --- a/pkg/nsx/services/realizestate/realize_state.go +++ b/pkg/nsx/services/realizestate/realize_state.go @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte return err } for _, result := range results.Results { - if *result.EntityType != entityType { + if entityType != "" && *result.EntityType != entityType { continue } if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED { diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 5189ac73e..1ef1e0678 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace, } func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string, - nsxVPC *model.Vpc) (*model.Vpc, + nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc, error) { vpc := &model.Vpc{} if nsxVPC != nil { @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return nil, nil } // for updating vpc case, use current vpc id, name - vpc = nsxVPC + *vpc = *nsxVPC } else { // for creating vpc case, fill in vpc properties based on networkconfig vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster) @@ -76,7 +76,11 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe }, } vpc.SiteInfos = siteInfos - vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled} + loadBalancerVPCEndpointEnabled := false + if useAVILB { + loadBalancerVPCEndpointEnabled = true + } + vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled} vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID) } @@ -89,3 +93,17 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return vpc, nil } + +func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) { + lbs := &model.LBService{} + lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster) + // LBS id should equal VPC id + lbs.Id = common.String(string(nsObj.GetUID())) + lbs.DisplayName = &lbsName + // TODO(gran) do we need "created_for" and "lb_t1_link_ip" tag? + lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + lbs.Size = &lbsSize + lbs.ConnectivityPath = &vpcPath + lbs.RelaxScaleValidation = relaxScaleValidation + return lbs, nil +} diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 6c5d90610..4b3ca419d 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -13,6 +13,8 @@ func keyFunc(obj interface{}) (string, error) { switch v := obj.(type) { case *model.Vpc: return *v.Id, nil + case *model.LBService: + return *v.Id, nil case *model.IpAddressBlock: return generateIPBlockKey(*v), nil default: @@ -27,6 +29,8 @@ func indexFunc(obj interface{}) ([]string, error) { switch o := obj.(type) { case *model.Vpc: return filterTag(o.Tags), nil + case *model.LBService: + return filterTag(o.Tags), nil case *model.IpAddressBlock: return filterTag(o.Tags), nil default: @@ -148,6 +152,41 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB return block } +// LBSStore is a store for LBS +type LBSStore struct { + common.ResourceStore +} + +func (ls *LBSStore) Apply(i interface{}) error { + if i == nil { + return nil + } + lbs := i.(*model.LBService) + if lbs.MarkedForDelete != nil && *lbs.MarkedForDelete { + err := ls.Delete(lbs) + log.V(1).Info("delete LBS from store", "LBS", lbs) + if err != nil { + return err + } + } else { + err := ls.Add(lbs) + log.V(1).Info("add LBS to store", "LBS", lbs) + if err != nil { + return err + } + } + return nil +} + +func (ls *LBSStore) GetByKey(key string) *model.LBService { + obj := ls.ResourceStore.GetByKey(key) + if obj != nil { + lbs := obj.(*model.LBService) + return lbs + } + return nil +} + // keyFuncAVI is used to get the key of a AVI rule related resource func keyFuncAVI(obj interface{}) (string, error) { switch v := obj.(type) { diff --git a/pkg/nsx/services/vpc/store_test.go b/pkg/nsx/services/vpc/store_test.go index 2e6601615..c830d42df 100644 --- a/pkg/nsx/services/vpc/store_test.go +++ b/pkg/nsx/services/vpc/store_test.go @@ -32,20 +32,6 @@ func (qIface *fakeQueryClient) List(_ string, _ *string, _ *string, _ *int64, _ }, nil } -func Test_IndexFunc(t *testing.T) { - mId, mTag, mScope := "test_id", "test_tag", "nsx-op/namespace_uid" - v := &model.Vpc{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - } - t.Run("1", func(t *testing.T) { - got, _ := indexFunc(v) - if !reflect.DeepEqual(got, []string{"test_tag"}) { - t.Errorf("NSCRUIDScopeIndexFunc() = %v, want %v", got, model.Tag{Tag: &mTag, Scope: &mScope}) - } - }) -} - func Test_filterTag(t *testing.T) { mTag, mScope := "test_tag", "nsx-op/namespace_uid" mTag2, mScope2 := "test_tag", "nsx" @@ -74,17 +60,6 @@ func Test_filterTag(t *testing.T) { } } -func Test_KeyFunc(t *testing.T) { - Id := "test_id" - v := &model.Vpc{Id: &Id} - t.Run("1", func(t *testing.T) { - got, _ := keyFunc(v) - if got != "test_id" { - t.Errorf("keyFunc() = %v, want %v", got, "test_id") - } - }) -} - func Test_InitializeVPCStore(t *testing.T) { config2 := nsx.NewConfig("localhost", "1", "1", []string{}, 10, 3, 20, 20, true, true, true, ratelimiter.AIMD, nil, nil, []string{}) cluster, _ := nsx.NewCluster(config2) @@ -337,3 +312,103 @@ func TestSecurityPolicyStore_GetByKey(t *testing.T) { sp = spStore.GetByKey(path2) assert.Equal(t, sp.Path, sp2.Path) } + +func Test_keyFunc(t *testing.T) { + id := "test_id" + type args struct { + obj interface{} + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "vpc", + args: args{obj: &model.Vpc{Id: &id}}, + want: id, + wantErr: assert.NoError, + }, + { + name: "lbs", + args: args{obj: &model.LBService{Id: &id}}, + want: id, + wantErr: assert.NoError, + }, + { + name: "invalid", + args: args{obj: &model.AntreaTraceflowConfig{Id: &id}}, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := keyFunc(tt.args.obj) + if !tt.wantErr(t, err, fmt.Sprintf("keyFunc(%v)", tt.args.obj)) { + return + } + assert.Equalf(t, tt.want, got, "keyFunc(%v)", tt.args.obj) + }) + } +} + +func Test_indexFunc(t *testing.T) { + mId, mTag, mScope := "test_id", "test_tag", "nsx-op/namespace_uid" + type args struct { + obj interface{} + } + tests := []struct { + name string + args args + want []string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "vpc", + args: args{obj: &model.Vpc{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{"test_tag"}, + wantErr: assert.NoError, + }, + { + name: "lbs", + args: args{obj: &model.LBService{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{"test_tag"}, + wantErr: assert.NoError, + }, + { + name: "lbsnotag", + args: args{obj: &model.LBService{ + Id: &mId, + Tags: []model.Tag{}, + }}, + want: []string{}, + wantErr: assert.NoError, + }, + { + name: "invalid", + args: args{obj: &model.AntreaTraceflowConfig{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{}, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := indexFunc(tt.args.obj) + if !tt.wantErr(t, err, fmt.Sprintf("indexFunc(%v)", tt.args.obj)) { + return + } + assert.Equalf(t, tt.want, got, "indexFunc(%v)", tt.args.obj) + }) + } +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 415b92773..f139c6fff 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -15,9 +15,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" @@ -30,6 +30,7 @@ const ( AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule" VPCAviSEGroupId = "avi-se-vms" VpcDefaultSecurityPolicyId = "default-layer3-section" + VPCKey = "/orgs/%s/projects/%s/vpcs/%s" GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s" SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s" RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s" @@ -41,8 +42,9 @@ var ( ResourceTypeVPC = common.ResourceTypeVpc NewConverter = common.NewConverter - MarkedForDelete = true - enableAviAllowRule = false + MarkedForDelete = true + enableAviAllowRule = false + EnforceRevisionCheckParam = false ) type VPCNetworkInfoStore struct { @@ -58,6 +60,7 @@ type VPCNsNetworkConfigStore struct { type VPCService struct { common.Service VpcStore *VPCStore + LbsStore *LBSStore IpblockStore *IPBlockStore VPCNetworkConfigStore VPCNetworkInfoStore VPCNSNetworkConfigStore VPCNsNetworkConfigStore @@ -161,6 +164,10 @@ func InitializeVPC(service common.Service) (*VPCService, error) { Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), BindingType: model.VpcBindingType(), }} + VPCService.LbsStore = &LBSStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBServiceBindingType(), + }} VPCService.IpblockStore = &IPBlockStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ @@ -173,8 +180,10 @@ func InitializeVPC(service common.Service) (*VPCService, error) { VPCService.VPCNSNetworkConfigStore = VPCNsNetworkConfigStore{ VPCNSNetworkConfigMap: make(map[string]string), } - //initialize vpc store and ip blocks store + //initialize vpc store, lbs store and ip blocks store go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeVpc, nil, VPCService.VpcStore) + wg.Add(1) + go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeLBService, nil, VPCService.LbsStore) go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeIPBlock, nil, VPCService.IpblockStore) //initalize avi rule related store @@ -251,6 +260,10 @@ func (s *VPCService) DeleteVPC(path string) error { err = nsxutil.NSXApiError(err) return err } + lbs := s.LbsStore.GetByKey(pathInfo.VPCID) + if lbs != nil { + s.LbsStore.Delete(lbs) + } vpc.MarkedForDelete = &MarkedForDelete if err := s.VpcStore.Apply(vpc); err != nil { return err @@ -556,7 +569,7 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * nsxVPC = nil } - createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC) + createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC, s.NSXConfig.NsxConfig.UseAVILB) if err != nil { log.Error(err, "failed to build NSX VPC object") return nil, nil, err @@ -568,8 +581,34 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return existingVPC[0], &nc, nil } + // build NSX LBS + var createdLBS *model.LBService + // TODO(gran) use switch to enable/disable NSXLB and AVI + if s.NSXConfig.NsxConfig.NSXLBEnabled() { + lbsSize := s.NSXConfig.NsxConfig.ServiceSize + if lbsSize == "" { + lbsSize = config.LB_SERVICE_SIZE_SMALL + } + // TODO(gran) Uncomment after #618 merged + //if nc.LbServiceSize != "" { + // lbsSize = nc.LbServiceSize + //} + vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name) + var relaxScaleValidation *bool + if s.NSXConfig.NsxConfig.RelaxScaleValidaion { + relaxScaleValidation = common.Bool(true) + } + createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) + } + // build HAPI request + orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NsxtProject, createdVpc, createdLBS) + if err != nil { + log.Error(err, "failed to build HAPI request") + return nil, nil, err + } + log.Info("creating NSX VPC", "VPC", *createdVpc.Id) - err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc) + err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &EnforceRevisionCheckParam) err = nsxutil.NSXApiError(err) if err != nil { log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace) @@ -596,12 +635,14 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return nil, nil, err } + log.V(2).Info("check VPC realization state", "VPC", *createdVpc.Id) realizeService := realizestate.InitializeRealizeState(s.Service) - if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newVpc.Path, "RealizedLogicalRouter"); err != nil { + if err = realizeService.CheckRealizeState(util.NSXTDefaultRetry, *newVpc.Path, "RealizedLogicalRouter"); err != nil { log.Error(err, "failed to check VPC realization state", "VPC", *createdVpc.Id) if realizestate.IsRealizeStateError(err) { log.Error(err, "the created VPC is in error realization state, cleaning the resource", "VPC", *createdVpc.Id) // delete the nsx vpc object and re-created in next loop + // TODO(gran) DeleteVPC will check VpcStore but new Vpc is not in store at this moment. Is it correct? if err := s.DeleteVPC(*newVpc.Path); err != nil { log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) return nil, nil, err @@ -611,6 +652,32 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * } s.VpcStore.Add(&newVpc) + + // Check LBS realization + if createdLBS != nil { + newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id) + if err != nil { + log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id) + return nil, nil, err + } + s.LbsStore.Add(&newLBS) + + log.V(2).Info("check LBS realization state", "LBS", *createdLBS.Id) + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(util.NSXTLBVSDefaultRetry, *newLBS.Path, ""); err != nil { + log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id) + // delete the nsx vpc object and re-created in next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, nil, err + } + } + return nil, nil, err + } + } + return &newVpc, &nc, nil } @@ -902,3 +969,11 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { } return VPCInfoList } + +func (s *VPCService) GetNSXLBSPath(lbsId string) string { + vpcLBS := s.LbsStore.GetByKey(lbsId) + if vpcLBS == nil { + return "" + } + return *vpcLBS.Path +} diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go new file mode 100644 index 000000000..8b16e771b --- /dev/null +++ b/pkg/nsx/services/vpc/wrap.go @@ -0,0 +1,41 @@ +package vpc + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) { + if lbs != nil { + var vpcChildren []*data.StructValue + childrenLBS, err := s.WrapLBS(lbs) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, childrenLBS...) + vpc.Children = vpcChildren + } + var projectChildren []*data.StructValue + childrenVPC, err := s.WrapVPC(vpc) + if err != nil { + return nil, err + } + projectChildren = append(projectChildren, childrenVPC...) + + var orgChildren []*data.StructValue + childrenProject, err := s.WrapProject(nsxtProject, projectChildren) + if err != nil { + return nil, err + } + orgChildren = append(orgChildren, childrenProject...) + + var orgRootChildren []*data.StructValue + childrenOrg, err := s.WrapOrg(org, orgChildren) + if err != nil { + return nil, err + } + orgRootChildren = append(orgRootChildren, childrenOrg...) + + orgRoot, _ := s.WrapOrgRoot(orgRootChildren) + return orgRoot, nil +} diff --git a/pkg/nsx/util/utils.go b/pkg/nsx/util/utils.go index 2c26f18d4..621014934 100644 --- a/pkg/nsx/util/utils.go +++ b/pkg/nsx/util/utils.go @@ -561,6 +561,8 @@ func CasttoPointer(obj interface{}) interface{} { return &v case model.Vpc: return &v + case model.LBService: + return &v case model.IpAddressPoolBlockSubnet: return &v case model.Group: diff --git a/pkg/util/retry.go b/pkg/util/retry.go new file mode 100644 index 000000000..131529a0e --- /dev/null +++ b/pkg/util/retry.go @@ -0,0 +1,21 @@ +package util + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var NSXTDefaultRetry = wait.Backoff{ + Steps: 10, + Duration: 500 * time.Millisecond, + Factor: 1.0, + Jitter: 0.1, +} + +var NSXTLBVSDefaultRetry = wait.Backoff{ + Steps: 60, + Duration: 500 * time.Millisecond, + Factor: 1.0, + Jitter: 0.1, +}