Merge pull request #369 from TaoZou1/avirule

Add AVI allow rule when VPC created

pkg/clean/clean.go

@@ -58,7 +58,6 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er
 		NSXClient: nsxClient,
 		NSXConfig: cf,
 	}
-
 	vpcService, vpcErr := vpc.InitializeVPC(commonService)
 	commonctl.ServiceMediator.VPCService = vpcService
 

pkg/controllers/vpc/vpc_controller.go

@@ -70,6 +70,12 @@ func (r *VPCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 			updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client)
 			return common.ResultRequeueAfter10sec, err
 		}
+		err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
+		if err != nil {
+			log.Error(err, "operate failed, would retry exponentially", "VPC", req.NamespacedName)
+			updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client)
+			return common.ResultRequeueAfter10sec, err
+		}
 
 		snatIP, path, cidr := "", "", ""
 		// currently, auto snat is not exposed, and use default value True

pkg/nsx/client.go

@@ -23,6 +23,7 @@ import (
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra/realized_state"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs"
 	nat "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/nat"
+	vpc_sp "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/security_policies"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ip_pools"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ports"
@@ -39,10 +40,11 @@ const (
 	ServiceAccountRestore
 	ServiceAccountCertRotation
 	StaticRoute
+	VpcAviRule
 	AllFeatures
 )
 
-var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE"}
+var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE", "VPC_AVI_RULE"}
 
 type Client struct {
 	NsxConfig     *config.NSXOperatorConfig
@@ -63,6 +65,10 @@ type Client struct {
 	PrincipalIdentitiesClient  trust_management.PrincipalIdentitiesClient
 	WithCertificateClient      principal_identities.WithCertificateClient
 
+	// for AVI security policy rule
+	VPCSecurityClient vpcs.SecurityPoliciesClient
+	VPCRuleClient     vpc_sp.RulesClient
+
 	OrgRootClient       nsx_policy.OrgRootClient
 	ProjectInfraClient  projects.InfraClient
 	VPCClient           projects.VpcsClient
@@ -94,12 +100,8 @@ type NSXHealthChecker struct {
 }
 
 type NSXVersionChecker struct {
-	cluster                           *Cluster
-	securityPolicySupported           bool
-	nsxServiceAccountSupported        bool
-	nsxServiceAccountRestoreSupported bool
-	vpcSupported                      bool
-	featureSupported                  [AllFeatures]bool
+	cluster          *Cluster
+	featureSupported [AllFeatures]bool
 }
 
 func (ck *NSXHealthChecker) CheckNSXHealth(req *http.Request) error {
@@ -158,6 +160,9 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
 
+	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
+	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
+
 	nsxChecker := &NSXHealthChecker{
 		cluster: cluster,
 	}
@@ -193,6 +198,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		PortClient:         portClient,
 		PortStateClient:    portStateClient,
 		SubnetStatusClient: subnetStatusClient,
+		VPCSecurityClient:  vpcSecurityClient,
+		VPCRuleClient:      vpcRuleClient,
 
 		NSXChecker:          *nsxChecker,
 		NSXVerChecker:       *nsxVersionChecker,
@@ -215,6 +222,10 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		err := errors.New("NSXServiceAccountRestore feature support check failed")
 		log.Error(err, "initial NSX version check for NSXServiceAccountRestore got error")
 	}
+	if !nsxClient.NSXCheckVersion(VpcAviRule) {
+		err := errors.New("VpcAviRule feature support check failed")
+		log.Error(err, "initial NSX version check for VpcAviRule got error")
+	}
 	if !nsxClient.NSXCheckVersion(ServiceAccountCertRotation) {
 		err := errors.New("ServiceAccountCertRotation feature support check failed")
 		log.Error(err, "initial NSX version check for ServiceAccountCertRotation got error")
@@ -247,3 +258,7 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 	client.NSXVerChecker.featureSupported[feature] = true
 	return true
 }
+
+func (client *Client) FeatureEnabled(feature int) bool {
+	return client.NSXVerChecker.featureSupported[feature] == true
+}

pkg/nsx/cluster.go

@@ -331,6 +331,9 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	case ServiceAccountCertRotation:
 		minVersion = nsx413Version
 		validFeature = true
+	case VpcAviRule:
+		minVersion = nsx411Version
+		validFeature = true
 	}
 
 	if validFeature {

pkg/nsx/services/common/store.go

@@ -132,40 +132,13 @@ func (service *Service) InitializeVPCResourceStore(wg *sync.WaitGroup, fatalErro
 	service.InitializeCommonStore(wg, fatalErrors, org, project, resourceTypeValue, tags, store)
 }
 
-// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore
-func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) {
-	defer wg.Done()
-
-	tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1)
-	tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1)
-	tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue)
-
-	for _, tag := range tags {
-		tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1)
-		tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey)
-		if tag.Tag != nil {
-			tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1)
-			tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue)
-		}
-	}
-
-	resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue)
-	queryParam := resourceParam + " AND " + tagParam
-
-	if org != "" || project != "" {
-		// QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A",
-		//"path%25%3A" would fail to get response. Hack it here.
-		path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*"
-		pathUnescape, _ := url.PathUnescape("path%3A")
-		queryParam += " AND " + pathUnescape + path
-	}
-	queryParam += " AND marked_for_delete:false"
+type Filter func(interface{}) *data.StructValue
 
+func (service *Service) SearchResource(resourceTypeValue string, queryParam string, store Store, filter Filter) (uint64, error) {
 	var cursor *string = nil
 	count := uint64(0)
 	for {
 		var err error
-
 		var results []*data.StructValue
 		var resultCount *int64
 		if store.IsPolicyAPI() {
@@ -181,18 +154,22 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch
 			resultCount = response.ResultCount
 			err = searchEerr
 		}
+
 		err = TransError(err)
 		if _, ok := err.(nsxutil.PageMaxError); ok == true {
 			DecrementPageSize(Int64(PageSize))
 			continue
 		}
 		if err != nil {
-			fatalErrors <- err
+			return count, err
 		}
 		for _, entity := range results {
+			if filter != nil {
+				entity = filter(entity)
+			}
 			err = store.TransResourceToStore(entity)
 			if err != nil {
-				fatalErrors <- err
+				return count, err
 			}
 			count++
 		}
@@ -204,5 +181,44 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch
 			break
 		}
 	}
+	return count, nil
+}
+
+// PopulateResourcetoStore is the method used by populating resources created not by nsx-operator
+func (service *Service) PopulateResourcetoStore(wg *sync.WaitGroup, fatalErrors chan error, resourceTypeValue string, queryParam string, store Store, filter Filter) {
+	defer wg.Done()
+	count, err := service.SearchResource(resourceTypeValue, queryParam, store, filter)
+	if err != nil {
+		fatalErrors <- err
+	}
 	log.Info("initialized store", "resourceType", resourceTypeValue, "count", count)
 }
+
+// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore
+func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) {
+	tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1)
+	tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1)
+	tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue)
+
+	for _, tag := range tags {
+		tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1)
+		tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey)
+		if tag.Tag != nil {
+			tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1)
+			tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue)
+		}
+	}
+
+	resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue)
+	queryParam := resourceParam + " AND " + tagParam
+
+	if org != "" || project != "" {
+		// QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A",
+		//"path%25%3A" would fail to get response. Hack it here.
+		path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*"
+		pathUnescape, _ := url.PathUnescape("path%3A")
+		queryParam += " AND " + pathUnescape + path
+	}
+	queryParam += " AND marked_for_delete:false"
+	service.PopulateResourcetoStore(wg, fatalErrors, resourceTypeValue, queryParam, store, nil)
+}

pkg/nsx/services/common/types.go

@@ -62,6 +62,7 @@ const (
 	TagValueGroupScope              string = "scope"
 	TagValueGroupSrc                string = "source"
 	TagValueGroupDst                string = "destination"
+	TagValueGroupAvi                string = "avi"
 	AnnotationVPCNetworkConfig      string = "nsx.vmware.com/vpc_network_config"
 	AnnotationVPCName               string = "nsx.vmware.com/vpc_name"
 	AnnotationPodMAC                string = "nsx.vmware.com/mac"

pkg/nsx/services/vpc/store.go

@@ -147,3 +147,86 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB
 	block := indexResults[0].((model.IpAddressBlock))
 	return &block
 }
+
+// keyFuncAVI is used to get the key of a AVI rule related resource
+func keyFuncAVI(obj interface{}) (string, error) {
+	switch v := obj.(type) {
+	case model.Rule:
+		return *v.Path, nil
+	case model.SecurityPolicy:
+		return *v.Path, nil
+	case model.Group:
+		return *v.Path, nil
+	case model.IpAddressBlock:
+		return *v.Path, nil
+	default:
+		return "", errors.New("keyFunc doesn't support unknown type")
+	}
+}
+
+// AviRuleStore is a store for saving AVI related Rules in VPCs
+type AviRuleStore struct {
+	common.ResourceStore
+}
+
+func (ruleStore *AviRuleStore) Apply(i interface{}) error {
+	return nil
+}
+func (ruleStore *AviRuleStore) GetByKey(key string) *model.Rule {
+	obj := ruleStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		rule := obj.(model.Rule)
+		return &rule
+	}
+	return nil
+}
+
+// PubIPblockStore is a store to query external ip blocks cidr
+type PubIPblockStore struct {
+	common.ResourceStore
+}
+
+func (ipBlockStore *PubIPblockStore) Apply(i interface{}) error {
+	return nil
+}
+func (ipBlockStore *PubIPblockStore) GetByKey(key string) *model.IpAddressBlock {
+	obj := ipBlockStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		ipblock := obj.(model.IpAddressBlock)
+		return &ipblock
+	}
+	return nil
+}
+
+type AviGroupStore struct {
+	common.ResourceStore
+}
+
+func (groupStore *AviGroupStore) Apply(i interface{}) error {
+	return nil
+}
+func (groupStore *AviGroupStore) GetByKey(key string) *model.Group {
+	obj := groupStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		group := obj.(model.Group)
+		return &group
+	}
+	return nil
+}
+
+type AviSecurityPolicyStore struct {
+	common.ResourceStore
+}
+
+func (securityPolicyStore *AviSecurityPolicyStore) Apply(i interface{}) error {
+	return nil
+}
+
+func (securityPolicyStore *AviSecurityPolicyStore) GetByKey(key string) *model.SecurityPolicy {
+	obj := securityPolicyStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		sp := obj.(model.SecurityPolicy)
+		return &sp
+	}
+	return nil
+}