Change DefaultSubnetAccessMode to DefaultAccessModeForPod

As we discussed, we want to set default Pod Subnet type in Namespace settings, and
always keep VMs default to "Private" Subnets.
Also, besides "Public" and "Private" access mode, also add "Project" and "Isolated"
for user to choose.

build/yaml/crd/nsx.vmware.com_ippools.yaml

@@ -172,10 +172,12 @@ spec:
                   type: object
                 type: array
               type:
-                description: Type defines the type of this IPPool, Public or Private.
+                description: Type defines the type of this IPPool, Public, Private
+                  or Project.
                 enum:
                 - Public
                 - Private
+                - Project
                 type: string
             type: object
           status:

build/yaml/crd/nsx.vmware.com_subnets.yaml

@@ -85,6 +85,7 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_subnetsets.yaml

@@ -85,6 +85,7 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml

@@ -61,12 +61,13 @@ spec:
                 description: Default size of Subnet based upon estimated workload
                   count. Defaults to 26.
                 type: integer
-              defaultSubnetAccessMode:
-                description: DefaultSubnetAccessMode defines the access mode of the
-                  default SubnetSet for PodVM and VM. Must be Public or Private.
+              defaultPodSubnetAccessMode:
+                description: DefaultPodSubnetAccessMode defines the access mode of
+                  the default SubnetSet for PodVM. Must be Public or Private.
                 enum:
                 - Public
                 - Private
+                - Project
                 type: string
               edgeClusterPath:
                 description: Edge cluster path on which the networking elements will

go.mod

@@ -2,6 +2,11 @@ module github.com/vmware-tanzu/nsx-operator
 
 go 1.21
 
+replace (
+	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+)
+
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0

pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,10 +37,15 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultPodSubnetAccessMode defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project
+	DefaultPodSubnetAccessMode string `json:"defaultPodSubnetAccessMode,omitempty"`
+	// ShortID specifies Identifier to use when displaying VPC context in logs.
+	// Less than equal to 8 characters.
+	// +kubebuilder:validation:MaxLength=8
+	// +optional
+	ShortID string `json:"shortID,omitempty"`
 }
 
 // VPCNetworkConfigurationStatus defines the observed state of VPCNetworkConfiguration

pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/apis/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,10 +37,10 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultPodSubnetAccessMode defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project
+	DefaultPodSubnetAccessMode string `json:"defaultPodSubnetAccessMode,omitempty"`
 	// ShortID specifies Identifier to use when displaying VPC context in logs.
 	// Less than equal to 8 characters.
 	// +kubebuilder:validation:MaxLength=8

pkg/apis/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/controllers/ippool/ippool_controller.go

@@ -141,7 +141,7 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			updateFail(r, &ctx, obj, &err)
 			return resultRequeue, err
 		}
-		obj.Spec.Type = vpcNetworkConfig.DefaultSubnetAccessMode
+		obj.Spec.Type = "Private"
 	}
 
 	if obj.ObjectMeta.DeletionTimestamp.IsZero() {

pkg/controllers/namespace/namespace_controller.go

@@ -60,18 +60,6 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec
 		log.Info("vpc cr already exist, skip creating", "VPC", vpcs.Items[0].Name)
 		return &vpcs.Items[0], nil
 	}
-	nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
-	if !ncExist {
-		message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
-	if !r.VPCService.ValidateNetworkConfig(nc) {
-		// if network config is not valid, no need to retry, skip processing
-		message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
 
 	// create vpc cr with existing vpc network config
 	vpcCR := BuildVPCCR(ns, ncName, vpcName)
@@ -91,7 +79,7 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec
 	return vpcCR, nil
 }
 
-func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
+func (r *NamespaceReconciler) createDefaultSubnetSet(ns string, defaultPodAccessMode string) error {
 	defaultSubnetSets := map[string]string{
 		types.DefaultVMSubnetSet:  types.LabelDefaultVMSubnetSet,
 		types.DefaultPodSubnetSet: types.LabelDefaultPodSubnetSet,
@@ -127,6 +115,12 @@ func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
 					},
 				},
 			}
+			if name == types.DefaultVMSubnetSet {
+				// use "Private" type for VM
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
+			} else if name == types.DefaultPodSubnetSet {
+				obj.Spec.AccessMode = v1alpha1.AccessMode(defaultPodAccessMode)
+			}
 			if err := r.Client.Create(context.Background(), obj); err != nil {
 				return err
 			}
@@ -246,11 +240,23 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 				return common.ResultRequeueAfter10sec, nil
 			}
 		}
+		nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
+		if !ncExist {
+			message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
+		if !r.VPCService.ValidateNetworkConfig(nc) {
+			// if network config is not valid, no need to retry, skip processing
+			message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
 
 		if _, err := r.createVPCCR(&ctx, obj, ns, ncName, createVpcName); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
-		if err := r.createDefaultSubnetSet(ns); err != nil {
+		if err := r.createDefaultSubnetSet(ns, nc.DefaultPodSubnetAccessMode); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
 		return common.ResultNormal, nil

pkg/controllers/subnet/subnet_controller.go

@@ -80,7 +80,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 				return ResultRequeue, err
 			}
 			if obj.Spec.AccessMode == "" {
-				obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 			}
 			if obj.Spec.IPv4SubnetSize == 0 {
 				obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize

pkg/controllers/subnetset/subnetset_controller.go

@@ -71,7 +71,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 					return ResultRequeue, err
 				}
 				if obj.Spec.AccessMode == "" {
-					obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+					obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 				}
 				if obj.Spec.IPv4SubnetSize == 0 {
 					obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize

pkg/controllers/vpc/vpc_utils.go

@@ -153,17 +153,17 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*type
 	}
 
 	ninfo := &types.VPCNetworkConfigInfo{
-		IsDefault:               isDefaultNetworkConfigCR(vpcConfigCR),
-		Org:                     org,
-		Name:                    vpcConfigCR.Name,
-		DefaultGatewayPath:      vpcConfigCR.Spec.DefaultGatewayPath,
-		EdgeClusterPath:         vpcConfigCR.Spec.EdgeClusterPath,
-		NsxtProject:             project,
-		ExternalIPv4Blocks:      vpcConfigCR.Spec.ExternalIPv4Blocks,
-		PrivateIPv4CIDRs:        vpcConfigCR.Spec.PrivateIPv4CIDRs,
-		DefaultIPv4SubnetSize:   vpcConfigCR.Spec.DefaultIPv4SubnetSize,
-		DefaultSubnetAccessMode: vpcConfigCR.Spec.DefaultSubnetAccessMode,
-		ShortID:                 vpcConfigCR.Spec.ShortID,
+		IsDefault:                  isDefaultNetworkConfigCR(vpcConfigCR),
+		Org:                        org,
+		Name:                       vpcConfigCR.Name,
+		DefaultGatewayPath:         vpcConfigCR.Spec.DefaultGatewayPath,
+		EdgeClusterPath:            vpcConfigCR.Spec.EdgeClusterPath,
+		NsxtProject:                project,
+		ExternalIPv4Blocks:         vpcConfigCR.Spec.ExternalIPv4Blocks,
+		PrivateIPv4CIDRs:           vpcConfigCR.Spec.PrivateIPv4CIDRs,
+		DefaultIPv4SubnetSize:      vpcConfigCR.Spec.DefaultIPv4SubnetSize,
+		DefaultPodSubnetAccessMode: vpcConfigCR.Spec.DefaultPodSubnetAccessMode,
+		ShortID:                    vpcConfigCR.Spec.ShortID,
 	}
 	return ninfo, nil
 }

pkg/controllers/vpc/vpc_utils_test.go

@@ -74,22 +74,22 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 	assert.NotNil(t, e)
 
 	spec1 := v1alpha1.VPCNetworkConfigurationSpec{
-		DefaultGatewayPath:      "test-gw-path-1",
-		EdgeClusterPath:         "test-edge-path-1",
-		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
-		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
-		DefaultIPv4SubnetSize:   64,
-		DefaultSubnetAccessMode: "Public",
-		NSXTProject:             "/orgs/default/projects/nsx_operator_e2e_test",
+		DefaultGatewayPath:         "test-gw-path-1",
+		EdgeClusterPath:            "test-edge-path-1",
+		ExternalIPv4Blocks:         []string{"external-ipb-1", "external-ipb-2"},
+		PrivateIPv4CIDRs:           []string{"private-ipb-1", "private-ipb-2"},
+		DefaultIPv4SubnetSize:      64,
+		DefaultPodSubnetAccessMode: "Public",
+		NSXTProject:                "/orgs/default/projects/nsx_operator_e2e_test",
 	}
 	spec2 := v1alpha1.VPCNetworkConfigurationSpec{
-		DefaultGatewayPath:      "test-gw-path-2",
-		EdgeClusterPath:         "test-edge-path-2",
-		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
-		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
-		DefaultIPv4SubnetSize:   32,
-		DefaultSubnetAccessMode: "Private",
-		NSXTProject:             "/orgs/anotherOrg/projects/anotherProject",
+		DefaultGatewayPath:         "test-gw-path-2",
+		EdgeClusterPath:            "test-edge-path-2",
+		ExternalIPv4Blocks:         []string{"external-ipb-1", "external-ipb-2"},
+		PrivateIPv4CIDRs:           []string{"private-ipb-1", "private-ipb-2"},
+		DefaultIPv4SubnetSize:      32,
+		DefaultPodSubnetAccessMode: "Private",
+		NSXTProject:                "/orgs/anotherOrg/projects/anotherProject",
 	}
 	testCRD1 := v1alpha1.VPCNetworkConfiguration{
 		Spec: spec1,
@@ -134,7 +134,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 			assert.Equal(t, tt.org, nc.Org)
 			assert.Equal(t, tt.project, nc.NsxtProject)
 			assert.Equal(t, tt.subnetSize, nc.DefaultIPv4SubnetSize)
-			assert.Equal(t, tt.accessMode, nc.DefaultSubnetAccessMode)
+			assert.Equal(t, tt.accessMode, nc.DefaultPodSubnetAccessMode)
 			assert.Equal(t, tt.isDefault, nc.IsDefault)
 		})
 	}

pkg/nsx/services/common/types.go

@@ -194,15 +194,15 @@ type VPCResourceInfo struct {
 }
 
 type VPCNetworkConfigInfo struct {
-	IsDefault               bool
-	Org                     string
-	Name                    string
-	DefaultGatewayPath      string
-	EdgeClusterPath         string
-	NsxtProject             string
-	ExternalIPv4Blocks      []string
-	PrivateIPv4CIDRs        []string
-	DefaultIPv4SubnetSize   int
-	DefaultSubnetAccessMode string
-	ShortID                 string
+	IsDefault                  bool
+	Org                        string
+	Name                       string
+	DefaultGatewayPath         string
+	EdgeClusterPath            string
+	NsxtProject                string
+	ExternalIPv4Blocks         []string
+	PrivateIPv4CIDRs           []string
+	DefaultIPv4SubnetSize      int
+	DefaultPodSubnetAccessMode string
+	ShortID                    string
 }