Change DefaultSubnetAccessMode to DefaultAccessModeForPod

As we discussed, we want to set default Pod Subnet type in Namespace settings, and
always keep VMs default to "Private" Subnets.
Also, besides "Public" and "Private" access mode, also add "Project" and "Isolated"
for user to choose.

build/yaml/crd/nsx.vmware.com_ippools.yaml

@@ -172,10 +172,12 @@ spec:
                   type: object
                 type: array
               type:
-                description: Type defines the type of this IPPool, Public or Private.
+                description: Type defines the type of this IPPool, Public, Private
+                  or Project.
                 enum:
                 - Public
                 - Private
+                - Project
                 type: string
             type: object
           status:

build/yaml/crd/nsx.vmware.com_subnets.yaml

@@ -85,6 +85,8 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
+                - Isolated
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_subnetsets.yaml

@@ -85,6 +85,8 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
+                - Isolated
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml

@@ -53,6 +53,15 @@ spec:
               When a field is not set in a Namespace's VPCNetworkConfiguration, the
               Namespace will use the value in the default VPCNetworkConfiguration.
             properties:
+              defaultAccessModeForPod:
+                description: DefaultAccessModeForPod defines the access mode of the
+                  default SubnetSet for PodVM. Must be Public or Private.
+                enum:
+                - Public
+                - Private
+                - Project
+                - Isolated
+                type: string
               defaultGatewayPath:
                 description: PolicyPath of Tier0 or Tier0 VRF gateway.
                 type: string
@@ -61,13 +70,6 @@ spec:
                 description: Default size of Subnet based upon estimated workload
                   count. Defaults to 26.
                 type: integer
-              defaultSubnetAccessMode:
-                description: DefaultSubnetAccessMode defines the access mode of the
-                  default SubnetSet for PodVM and VM. Must be Public or Private.
-                enum:
-                - Public
-                - Private
-                type: string
               edgeClusterPath:
                 description: Edge cluster path on which the networking elements will
                   be created.

go.mod

@@ -2,6 +2,11 @@ module github.com/vmware-tanzu/nsx-operator
 
 go 1.21
 
+replace (
+	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+)
+
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0

pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project;Isolated
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project;Isolated
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,10 +37,15 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultAccessModeForPod defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project;Isolated
+	DefaultAccessModeForPod string `json:"defaultAccessModeForPod,omitempty"`
+	// ShortID specifies Identifier to use when displaying VPC context in logs.
+	// Less than equal to 8 characters.
+	// +kubebuilder:validation:MaxLength=8
+	// +optional
+	ShortID string `json:"shortID,omitempty"`
 }
 
 // VPCNetworkConfigurationStatus defines the observed state of VPCNetworkConfiguration

pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/apis/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project;Isolated
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project;Isolated
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,10 +37,10 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultAccessModeForPod defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project;Isolated
+	DefaultAccessModeForPod string `json:"defaultAccessModeForPod,omitempty"`
 	// ShortID specifies Identifier to use when displaying VPC context in logs.
 	// Less than equal to 8 characters.
 	// +kubebuilder:validation:MaxLength=8

pkg/apis/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/controllers/ippool/ippool_controller.go

@@ -141,7 +141,7 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			updateFail(r, &ctx, obj, &err)
 			return resultRequeue, err
 		}
-		obj.Spec.Type = vpcNetworkConfig.DefaultSubnetAccessMode
+		obj.Spec.Type = "Private"
 	}
 
 	if obj.ObjectMeta.DeletionTimestamp.IsZero() {

pkg/controllers/namespace/namespace_controller.go

@@ -60,18 +60,6 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec
 		log.Info("vpc cr already exist, skip creating", "VPC", vpcs.Items[0].Name)
 		return &vpcs.Items[0], nil
 	}
-	nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
-	if !ncExist {
-		message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
-	if !r.VPCService.ValidateNetworkConfig(nc) {
-		// if network config is not valid, no need to retry, skip processing
-		message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
 
 	// create vpc cr with existing vpc network config
 	vpcCR := BuildVPCCR(ns, ncName, vpcName)
@@ -91,7 +79,7 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec
 	return vpcCR, nil
 }
 
-func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
+func (r *NamespaceReconciler) createDefaultSubnetSet(ns string, defaultPodAccessMode string) error {
 	defaultSubnetSets := map[string]string{
 		types.DefaultVMSubnetSet:  types.LabelDefaultVMSubnetSet,
 		types.DefaultPodSubnetSet: types.LabelDefaultPodSubnetSet,
@@ -127,6 +115,12 @@ func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
 					},
 				},
 			}
+			if name == types.DefaultVMSubnetSet {
+				// use "Private" type for VM
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
+			} else if name == types.DefaultPodSubnetSet {
+				obj.Spec.AccessMode = v1alpha1.AccessMode(defaultPodAccessMode)
+			}
 			if err := r.Client.Create(context.Background(), obj); err != nil {
 				return err
 			}
@@ -246,11 +240,23 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 				return common.ResultRequeueAfter10sec, nil
 			}
 		}
+		nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
+		if !ncExist {
+			message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
+		if !r.VPCService.ValidateNetworkConfig(nc) {
+			// if network config is not valid, no need to retry, skip processing
+			message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
 
 		if _, err := r.createVPCCR(&ctx, obj, ns, ncName, createVpcName); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
-		if err := r.createDefaultSubnetSet(ns); err != nil {
+		if err := r.createDefaultSubnetSet(ns, nc.DefaultAccessModeForPod); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
 		return common.ResultNormal, nil

pkg/controllers/subnet/subnet_controller.go

@@ -80,7 +80,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 				return ResultRequeue, err
 			}
 			if obj.Spec.AccessMode == "" {
-				obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 			}
 			if obj.Spec.IPv4SubnetSize == 0 {
 				obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize

pkg/controllers/subnetset/subnetset_controller.go

@@ -71,7 +71,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 					return ResultRequeue, err
 				}
 				if obj.Spec.AccessMode == "" {
-					obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+					obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 				}
 				if obj.Spec.IPv4SubnetSize == 0 {
 					obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize

pkg/controllers/vpc/vpc_utils.go

@@ -162,7 +162,7 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*type
 		ExternalIPv4Blocks:      vpcConfigCR.Spec.ExternalIPv4Blocks,
 		PrivateIPv4CIDRs:        vpcConfigCR.Spec.PrivateIPv4CIDRs,
 		DefaultIPv4SubnetSize:   vpcConfigCR.Spec.DefaultIPv4SubnetSize,
-		DefaultSubnetAccessMode: vpcConfigCR.Spec.DefaultSubnetAccessMode,
+		DefaultAccessModeForPod: vpcConfigCR.Spec.DefaultAccessModeForPod,
 		ShortID:                 vpcConfigCR.Spec.ShortID,
 	}
 	return ninfo, nil

pkg/controllers/vpc/vpc_utils_test.go

@@ -79,7 +79,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
 		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
 		DefaultIPv4SubnetSize:   64,
-		DefaultSubnetAccessMode: "Public",
+		DefaultAccessModeForPod: "Public",
 		NSXTProject:             "/orgs/default/projects/nsx_operator_e2e_test",
 	}
 	spec2 := v1alpha1.VPCNetworkConfigurationSpec{
@@ -88,7 +88,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
 		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
 		DefaultIPv4SubnetSize:   32,
-		DefaultSubnetAccessMode: "Private",
+		DefaultAccessModeForPod: "Private",
 		NSXTProject:             "/orgs/anotherOrg/projects/anotherProject",
 	}
 	testCRD1 := v1alpha1.VPCNetworkConfiguration{
@@ -134,7 +134,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 			assert.Equal(t, tt.org, nc.Org)
 			assert.Equal(t, tt.project, nc.NsxtProject)
 			assert.Equal(t, tt.subnetSize, nc.DefaultIPv4SubnetSize)
-			assert.Equal(t, tt.accessMode, nc.DefaultSubnetAccessMode)
+			assert.Equal(t, tt.accessMode, nc.DefaultAccessModeForPod)
 			assert.Equal(t, tt.isDefault, nc.IsDefault)
 		})
 	}

pkg/nsx/services/common/types.go

@@ -203,6 +203,6 @@ type VPCNetworkConfigInfo struct {
 	ExternalIPv4Blocks      []string
 	PrivateIPv4CIDRs        []string
 	DefaultIPv4SubnetSize   int
-	DefaultSubnetAccessMode string
+	DefaultAccessModeForPod string
 	ShortID                 string
 }

test/e2e/nsx_subnet_test.go

@@ -27,7 +27,7 @@ const (
 	SubnetDeletionTimeout = 300 * time.Second
 )
 
-func verifySubnetSetCR(subnetSet string) bool {
+func verifySubnetSetCR(subnetSet string, subnetType string) bool {
 	vpcNetworkConfig, err := testData.crdClientset.NsxV1alpha1().VPCNetworkConfigurations().Get(context.TODO(), VPCNetworkConfigCRName, v1.GetOptions{})
 	if err != nil {
 		log.Printf("Failed to get VPCNetworkConfiguration %s: %v", VPCNetworkConfigCRName, err)
@@ -38,8 +38,14 @@ func verifySubnetSetCR(subnetSet string) bool {
 		log.Printf("Failed to get %s/%s: %s", E2ENamespace, subnetSet, err)
 		return false
 	}
-	if string(subnetSetCR.Spec.AccessMode) != vpcNetworkConfig.Spec.DefaultSubnetAccessMode {
-		log.Printf("AccessMode is %s, while it's expected to be %s", subnetSetCR.Spec.AccessMode, vpcNetworkConfig.Spec.DefaultSubnetAccessMode)
+	var desiredSubnetType string
+	if subnetType == "" {
+		desiredSubnetType = vpcNetworkConfig.Spec.DefaultAccessModeForPod
+	} else {
+		desiredSubnetType = subnetType
+	}
+	if string(subnetSetCR.Spec.AccessMode) != desiredSubnetType {
+		log.Printf("AccessMode is %s, while it's expected to be %s", subnetSetCR.Spec.AccessMode, desiredSubnetType)
 		return false
 	}
 	if subnetSetCR.Spec.IPv4SubnetSize != vpcNetworkConfig.Spec.DefaultIPv4SubnetSize {
@@ -74,8 +80,8 @@ func defaultSubnetSet(t *testing.T) {
 	assertNil(t, err)
 
 	// 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig.
-	assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet))
-	assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet))
+	assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet, "Private"))
+	assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet, ""))
 
 	portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_1.yaml")
 	err = applyYAML(portPath, E2ENamespace)
@@ -157,7 +163,7 @@ func userSubnetSet(t *testing.T) {
 	assertNil(t, err)
 
 	// 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig.
-	assertTrue(t, verifySubnetSetCR(UserSubnetSet))
+	assertTrue(t, verifySubnetSetCR(UserSubnetSet, "Private"))
 
 	portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_2.yaml")
 	err = applyYAML(portPath, E2ENamespace)
@@ -186,8 +192,8 @@ func sharedSubnetSet(t *testing.T) {
 	assertNil(t, err)
 
 	// 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig.
-	assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet))
-	assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet))
+	assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet, "Private"))
+	assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet, ""))
 
 	portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_3.yaml")
 	err = applyYAML(portPath, E2ENamespaceShared)