diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index c1e01f2e8..25078caae 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -15,12 +15,14 @@ import ( mpsearch "github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/search" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/trust_management" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/trust_management/principal_identities" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/domains" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/domains/security_policies" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/shares" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/sites/enforcement_points" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects" - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra" + project_infra "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra/realized_state" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/transit_gateways" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs" @@ -77,7 +79,7 @@ type Client struct { ProjectInfraClient projects.InfraClient VPCClient projects.VpcsClient VPCConnectivityProfilesClient projects.VpcConnectivityProfilesClient - IPBlockClient infra.IpBlocksClient + IPBlockClient project_infra.IpBlocksClient StaticRouteClient vpcs.StaticRoutesClient NATRuleClient nat.NatRulesClient VpcGroupClient vpcs.GroupsClient @@ -92,6 +94,12 @@ type Client struct { ProjectClient orgs.ProjectsClient TransitGatewayClient projects.TransitGatewaysClient TransitGatewayAttachmentClient transit_gateways.AttachmentsClient + CertificateClient infra.CertificatesClient + ShareClient infra.SharesClient + SharedResourceClient shares.ResourcesClient + LbAppProfileClient infra.LbAppProfilesClient + LbPersistenceProfilesClient infra.LbPersistenceProfilesClient + LbMonitorProfilesClient infra.LbMonitorProfilesClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -162,7 +170,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { projectClient := orgs.NewProjectsClient(restConnector(cluster)) vpcClient := projects.NewVpcsClient(restConnector(cluster)) vpcConnectivityProfilesClient := projects.NewVpcConnectivityProfilesClient(restConnector(cluster)) - ipBlockClient := infra.NewIpBlocksClient(restConnector(cluster)) + ipBlockClient := project_infra.NewIpBlocksClient(restConnector(cluster)) staticRouteClient := vpcs.NewStaticRoutesClient(restConnector(cluster)) natRulesClient := nat.NewNatRulesClient(restConnector(cluster)) vpcGroupClient := vpcs.NewGroupsClient(restConnector(cluster)) diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 0fecaffdb..91889032e 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -140,29 +140,37 @@ var ( ) var ( - ResourceType = "resource_type" - ResourceTypeInfra = "Infra" - ResourceTypeDomain = "Domain" - ResourceTypeSecurityPolicy = "SecurityPolicy" - ResourceTypeNetworkPolicy = "NetworkPolicy" - ResourceTypeGroup = "Group" - ResourceTypeRule = "Rule" - ResourceTypeIPBlock = "IpAddressBlock" - ResourceTypeOrgRoot = "OrgRoot" - ResourceTypeOrg = "Org" - ResourceTypeProject = "Project" - ResourceTypeVpc = "Vpc" - ResourceTypeSubnetPort = "VpcSubnetPort" - ResourceTypeVirtualMachine = "VirtualMachine" - ResourceTypeLBService = "LBService" - ResourceTypeShare = "Share" - ResourceTypeSharedResource = "SharedResource" - ResourceTypeChildSharedResource = "ChildSharedResource" - ResourceTypeChildShare = "ChildShare" - ResourceTypeChildRule = "ChildRule" - ResourceTypeChildGroup = "ChildGroup" - ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" - ResourceTypeChildResourceReference = "ChildResourceReference" + ResourceType = "resource_type" + ResourceTypeInfra = "Infra" + ResourceTypeDomain = "Domain" + ResourceTypeSecurityPolicy = "SecurityPolicy" + ResourceTypeNetworkPolicy = "NetworkPolicy" + ResourceTypeGroup = "Group" + ResourceTypeRule = "Rule" + ResourceTypeIPBlock = "IpAddressBlock" + ResourceTypeOrgRoot = "OrgRoot" + ResourceTypeOrg = "Org" + ResourceTypeProject = "Project" + ResourceTypeVpc = "Vpc" + ResourceTypeSubnetPort = "VpcSubnetPort" + ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" + ResourceTypeShare = "Share" + ResourceTypeSharedResource = "SharedResource" + ResourceTypeChildSharedResource = "ChildSharedResource" + ResourceTypeChildShare = "ChildShare" + ResourceTypeChildRule = "ChildRule" + ResourceTypeChildGroup = "ChildGroup" + ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" + ResourceTypeChildResourceReference = "ChildResourceReference" + ResourceTypeTlsCertificate = "TlsCertificate" + ResourceTypeLBHttpProfile = "LBHttpProfile" + ResourceTypeLBFastTcpProfile = "LBFastTcpProfile" + ResourceTypeLBFastUdpProfile = "LBFastUdpProfile" + ResourceTypeLBCookiePersistenceProfile = "LBCookiePersistenceProfile" + ResourceTypeLBSourceIpPersistenceProfile = "LBSourceIpPersistenceProfile" + ResourceTypeLBHttpMonitorProfile = "LBHttpMonitorProfile" + ResourceTypeLBTcpMonitorProfile = "LBTcpMonitorProfile" // ResourceTypeClusterControlPlane is used by NSXServiceAccountController ResourceTypeClusterControlPlane = "clustercontrolplane" diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 4b3ca419d..8a5601e0e 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -141,6 +141,15 @@ func (vs *VPCStore) GetByKey(key string) *model.Vpc { return nil } +// ResourceStore is a store to query nsx resource +type ResourceStore struct { + common.ResourceStore +} + +func (r *ResourceStore) Apply(i interface{}) error { + return nil +} + func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressBlock { indexResults, err := is.ResourceStore.Indexer.ByIndex(index, value) if err != nil || len(indexResults) == 0 { diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 02f59cccd..6ea4685e4 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -277,6 +277,204 @@ func (s *VPCService) DeleteVPC(path string) error { return nil } +func (s *VPCService) addClusterTag(query string) string { + tagScopeClusterKey := strings.Replace(common.TagScopeNCPCluster, "/", "\\/", -1) + tagScopeClusterValue := strings.Replace(s.NSXClient.NsxConfig.Cluster, ":", "\\:", -1) + tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) + return query + " AND " + tagParam +} + +func (s *VPCService) ListCert() []model.TlsCertificate { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.TlsCertificateBindingType(), + }} + query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeTlsCertificate) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource(common.ResourceTypeTlsCertificate, query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query certificate", "query", query) + } else { + log.V(1).Info("query certificate", "count", count) + } + certs := store.List() + certsSet := []model.TlsCertificate{} + for _, cert := range certs { + certsSet = append(certsSet, *cert.(*model.TlsCertificate)) + } + return certsSet +} + +func (s *VPCService) DeleteCert(id string) error { + certClient := s.NSXClient.CertificateClient + if err := certClient.Delete(id); err != nil { + return err + } + log.Info("successfully deleted NCP created certificate", "certificate", id) + return nil +} + +func (s *VPCService) ListShare() []model.Share { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.ShareBindingType(), + }} + query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeShare) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource(common.ResourceTypeShare, query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query share", "query", query) + } else { + log.V(1).Info("query share", "count", count) + } + shares := store.List() + sharesSet := []model.Share{} + for _, cert := range shares { + sharesSet = append(sharesSet, *cert.(*model.Share)) + } + return sharesSet +} + +func (s *VPCService) DeleteShare(shareId string) error { + shareClient := s.NSXClient.ShareClient + if err := shareClient.Delete(shareId); err != nil { + return err + } + log.Info("successfully deleted NCP created share", "share", shareId) + return nil +} + +func (s *VPCService) ListSharedResource() []model.SharedResource { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.SharedResourceBindingType(), + }} + query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeSharedResource) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource(common.ResourceTypeSharedResource, query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query sharedResource", "query", query) + } else { + log.V(1).Info("query sharedResource", "count", count) + } + sharedResources := store.List() + sharedResourcesSet := []model.SharedResource{} + for _, sharedResource := range sharedResources { + sharedResourcesSet = append(sharedResourcesSet, *sharedResource.(*model.SharedResource)) + } + return sharedResourcesSet +} + +func (s *VPCService) DeleteSharedResource(shareId, id string) error { + sharedResourceClient := s.NSXClient.SharedResourceClient + if err := sharedResourceClient.Delete(shareId, id); err != nil { + return err + } + log.Info("successfully deleted NCP created sharedResource", "shareId", shareId, "sharedResource", id) + return nil +} + +func (s *VPCService) ListLBAppProfile() []model.LBAppProfile { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBAppProfileBindingType(), + }} + query := fmt.Sprintf("(%s:%s OR %s:%s OR %s:%s)", + common.ResourceType, common.ResourceTypeLBHttpProfile, + common.ResourceType, common.ResourceTypeLBFastTcpProfile, + common.ResourceType, common.ResourceTypeLBFastUdpProfile) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource(common.ResourceTypeLBHttpProfile, query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query LBAppProfile", "query", query) + } else { + log.V(1).Info("query LBAppProfile", "count", count) + } + lbAppProfiles := store.List() + lbAppProfilesSet := []model.LBAppProfile{} + for _, lbAppProfile := range lbAppProfiles { + lbAppProfilesSet = append(lbAppProfilesSet, *lbAppProfile.(*model.LBAppProfile)) + } + return lbAppProfilesSet +} + +func (s *VPCService) DeleteLBAppProfile(id string) error { + lbAppProfileClient := s.NSXClient.LbAppProfileClient + boolValue := false + if err := lbAppProfileClient.Delete(id, &boolValue); err != nil { + return err + } + log.Info("successfully deleted NCP created lbAppProfile", "lbAppProfile", id) + return nil +} + +func (s *VPCService) ListLBPersistenceProfile() []model.LBPersistenceProfile { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBPersistenceProfileBindingType(), + }} + query := fmt.Sprintf("(%s:%s OR %s:%s)", + common.ResourceType, common.ResourceTypeLBCookiePersistenceProfile, + common.ResourceType, common.ResourceTypeLBSourceIpPersistenceProfile) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource("", query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query LBPersistenceProfile", "query", query) + } else { + log.V(1).Info("query LBPersistenceProfile", "count", count) + } + lbPersistenceProfiles := store.List() + lbPersistenceProfilesSet := []model.LBPersistenceProfile{} + for _, lbPersistenceProfile := range lbPersistenceProfiles { + lbPersistenceProfilesSet = append(lbPersistenceProfilesSet, *lbPersistenceProfile.(*model.LBPersistenceProfile)) + } + return lbPersistenceProfilesSet +} + +func (s *VPCService) DeleteLBPersistenceProfile(id string) error { + lbPersistenceProfilesClient := s.NSXClient.LbPersistenceProfilesClient + boolValue := false + if err := lbPersistenceProfilesClient.Delete(id, &boolValue); err != nil { + return err + } + log.Info("successfully deleted NCP created lbPersistenceProfile", "lbPersistenceProfile", id) + return nil +} + +func (s *VPCService) ListLBMonitorProfile() []model.LBMonitorProfile { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBMonitorProfileBindingType(), + }} + query := fmt.Sprintf("(%s:%s OR %s:%s)", + common.ResourceType, common.ResourceTypeLBHttpMonitorProfile, + common.ResourceType, common.ResourceTypeLBTcpMonitorProfile) + query = s.addClusterTag(query) + count, searcherr := s.SearchResource("", query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query LBMonitorProfile", "query", query) + } else { + log.V(1).Info("query LBMonitorProfile", "count", count) + } + lbMonitorProfiles := store.List() + lbMonitorProfilesSet := []model.LBMonitorProfile{} + for _, lbMonitorProfile := range lbMonitorProfiles { + lbMonitorProfilesSet = append(lbMonitorProfilesSet, *lbMonitorProfile.(*model.LBMonitorProfile)) + } + return lbMonitorProfilesSet +} + +func (s *VPCService) DeleteLBMonitorProfile(id string) error { + lbMonitorProfilesClient := s.NSXClient.LbMonitorProfilesClient + boolValue := false + //nolint:staticcheck // SA1019 ignore this! + if err := lbMonitorProfilesClient.Delete(id, &boolValue); err != nil { + return err + } + log.Info("successfully deleted NCP created lbMonitorProfile", "lbMonitorProfile", id) + return nil +} + func (s *VPCService) deleteIPBlock(path string) error { ipblockClient := s.NSXClient.IPBlockClient parts := strings.Split(path, "/") @@ -733,6 +931,86 @@ func (s *VPCService) Cleanup(ctx context.Context) error { } } + // Delete NCP created resources (share/sharedResources/cert/LBAppProfile/LBPersistentProfile + sharedResources := s.ListSharedResource() + log.Info("cleaning up sharedResources", "Count", len(sharedResources)) + for _, sharedResource := range sharedResources { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + parentPath := strings.Split(*sharedResource.ParentPath, "/") + shareId := parentPath[len(parentPath)-1] + if err := s.DeleteSharedResource(shareId, *sharedResource.Id); err != nil { + return err + } + } + } + shares := s.ListShare() + log.Info("cleaning up shares", "Count", len(shares)) + for _, share := range shares { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteShare(*share.Id); err != nil { + return err + } + } + } + + certs := s.ListCert() + log.Info("cleaning up certificates", "Count", len(certs)) + for _, cert := range certs { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteCert(*cert.Id); err != nil { + return err + } + } + } + + lbAppProfiles := s.ListLBAppProfile() + log.Info("cleaning up lbAppProfiles", "Count", len(lbAppProfiles)) + for _, lbAppProfile := range lbAppProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteLBAppProfile(*lbAppProfile.Id); err != nil { + return err + } + } + } + + lbPersistenceProfiles := s.ListLBPersistenceProfile() + log.Info("cleaning up lbPersistenceProfiles", "Count", len(lbPersistenceProfiles)) + for _, lbPersistenceProfile := range lbPersistenceProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteLBPersistenceProfile(*lbPersistenceProfile.Id); err != nil { + return err + } + } + } + + lbMonitorProfiles := s.ListLBMonitorProfile() + log.Info("cleaning up lbMonitorProfiles", "Count", len(lbMonitorProfiles)) + for _, lbMonitorProfile := range lbMonitorProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteLBMonitorProfile(*lbMonitorProfile.Id); err != nil { + return err + } + } + } + // We don't clean client_ssl_profile as client_ssl_profile is not created by ncp or nsx-operator return nil }