Change DefaultSubnetAccessMode to DefaultAccessModeForPod

As we discussed, we want to set default Pod Subnet type in Namespace settings, and
always keep VMs default to "Private" Subnets.
Also, besides "Public" and "Private" access mode, also add "Project" and "Isolated"
for user to choose.

Update

Signed-off-by: Wenqi Qiu <wenqiq@vmware.com>

.gitignore

@@ -2,4 +2,6 @@
 bin/
 .DS_Store
 go.work
-go.work.sum
+go.work.sum
+vendor/
+.golangci-bin/

build/yaml/crd/nsx.vmware.com_ippools.yaml

@@ -172,10 +172,12 @@ spec:
                   type: object
                 type: array
               type:
-                description: Type defines the type of this IPPool, Public or Private.
+                description: Type defines the type of this IPPool, Public, Private
+                  or Project.
                 enum:
                 - Public
                 - Private
+                - Project
                 type: string
             type: object
           status:

build/yaml/crd/nsx.vmware.com_subnets.yaml

@@ -85,6 +85,7 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_subnetsets.yaml

@@ -85,6 +85,7 @@ spec:
                 enum:
                 - Private
                 - Public
+                - Project
                 type: string
               advancedConfig:
                 description: Subnet advanced configuration.

build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml

@@ -61,12 +61,13 @@ spec:
                 description: Default size of Subnet based upon estimated workload
                   count. Defaults to 26.
                 type: integer
-              defaultSubnetAccessMode:
-                description: DefaultSubnetAccessMode defines the access mode of the
-                  default SubnetSet for PodVM and VM. Must be Public or Private.
+              defaultPodSubnetAccessMode:
+                description: DefaultPodSubnetAccessMode defines the access mode of
+                  the default SubnetSet for PodVM. Must be Public or Private.
                 enum:
                 - Public
                 - Private
+                - Project
                 type: string
               edgeClusterPath:
                 description: Edge cluster path on which the networking elements will

go.mod

@@ -3,17 +3,21 @@ module github.com/vmware-tanzu/nsx-operator
 go 1.21.9
 
 replace (
-	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
-        github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+)
+
+replace (
+	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
 )
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/deckarep/golang-set v1.8.0
 	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/zapr v1.2.4
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -53,7 +57,6 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gibson042/canonicaljson-go v1.0.3 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect

go.sum

@@ -177,8 +177,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
 github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
 github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
 github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=

pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,12 +37,12 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultPodSubnetAccessMode defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project
+	DefaultPodSubnetAccessMode string `json:"defaultPodSubnetAccessMode,omitempty"`
 	// ShortID specifies Identifier to use when displaying VPC context in logs.
-	// Less than or equal to 8 characters.
+	// Less than equal to 8 characters.
 	// +kubebuilder:validation:MaxLength=8
 	// +optional
 	ShortID string `json:"shortID,omitempty"`
@@ -64,9 +64,9 @@ type VPCInfo struct {
 
 // +genclient
 // +genclient:nonNamespaced
-//+kubebuilder:object:root=true
-//+kubebuilder:subresource:status
-//+kubebuilder:storageversion
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:storageversion
 
 // VPCNetworkConfiguration is the Schema for the vpcnetworkconfigurations API.
 // +kubebuilder:resource:scope="Cluster"
@@ -81,7 +81,7 @@ type VPCNetworkConfiguration struct {
 	Status VPCNetworkConfigurationStatus `json:"status,omitempty"`
 }
 
-//+kubebuilder:object:root=true
+// +kubebuilder:object:root=true
 
 // VPCNetworkConfigurationList contains a list of VPCNetworkConfiguration.
 type VPCNetworkConfigurationList struct {

pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/apis/v1alpha1/subnet_types.go

@@ -16,7 +16,7 @@ type SubnetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0

pkg/apis/v1alpha1/subnetset_types.go

@@ -14,7 +14,7 @@ type SubnetSetSpec struct {
 	// +kubebuilder:validation:Minimum:=16
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
-	// +kubebuilder:validation:Enum=Private;Public
+	// +kubebuilder:validation:Enum=Private;Public;Project
 	AccessMode AccessMode `json:"accessMode,omitempty"`
 	// Subnet advanced configuration.
 	AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"`

pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go

@@ -37,10 +37,10 @@ type VPCNetworkConfigurationSpec struct {
 	// Defaults to 26.
 	// +kubebuilder:default=26
 	DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"`
-	// DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM.
+	// DefaultPodSubnetAccessMode defines the access mode of the default SubnetSet for PodVM.
 	// Must be Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
-	DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"`
+	// +kubebuilder:validation:Enum=Public;Private;Project
+	DefaultPodSubnetAccessMode string `json:"defaultPodSubnetAccessMode,omitempty"`
 	// ShortID specifies Identifier to use when displaying VPC context in logs.
 	// Less than or equal to 8 characters.
 	// +kubebuilder:validation:MaxLength=8

pkg/apis/v1alpha2/ippool_types.go

@@ -36,8 +36,8 @@ type IPPoolList struct {
 
 // IPPoolSpec defines the desired state of IPPool.
 type IPPoolSpec struct {
-	// Type defines the type of this IPPool, Public or Private.
-	// +kubebuilder:validation:Enum=Public;Private
+	// Type defines the type of this IPPool, Public, Private or Project.
+	// +kubebuilder:validation:Enum=Public;Private;Project
 	// +optional
 	Type string `json:"type,omitempty"`
 	// Subnets defines set of subnets need to be allocated.

pkg/controllers/ippool/ippool_controller.go

@@ -141,7 +141,7 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			updateFail(r, &ctx, obj, &err)
 			return resultRequeue, err
 		}
-		obj.Spec.Type = vpcNetworkConfig.DefaultSubnetAccessMode
+		obj.Spec.Type = "Private"
 	}
 
 	if obj.ObjectMeta.DeletionTimestamp.IsZero() {

pkg/controllers/namespace/namespace_controller.go

@@ -57,18 +57,6 @@ func (r *NamespaceReconciler) createNetworkInfoCR(ctx *context.Context, obj clie
 		log.Info("networkInfo already exists", "networkInfo", networkInfos.Items[0].Name, "Namespace", ns)
 		return &networkInfos.Items[0], nil
 	}
-	nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
-	if !ncExist {
-		message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
-	if !r.VPCService.ValidateNetworkConfig(nc) {
-		// if network config is not valid, no need to retry, skip processing
-		message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
-		r.namespaceError(ctx, obj, message, nil)
-		return nil, errors.New(message)
-	}
 
 	// create networkInfo cr with existing vpc network config
 	log.V(2).Info("building networkInfo", "ns", ns)
@@ -96,7 +84,7 @@ func (r *NamespaceReconciler) createNetworkInfoCR(ctx *context.Context, obj clie
 	return networkInfoCR, nil
 }
 
-func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
+func (r *NamespaceReconciler) createDefaultSubnetSet(ns string, defaultPodAccessMode string) error {
 	defaultSubnetSets := map[string]string{
 		types.DefaultVMSubnetSet:  types.LabelDefaultVMSubnetSet,
 		types.DefaultPodSubnetSet: types.LabelDefaultPodSubnetSet,
@@ -132,6 +120,12 @@ func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error {
 					},
 				},
 			}
+			if name == types.DefaultVMSubnetSet {
+				// use "Private" type for VM
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
+			} else if name == types.DefaultPodSubnetSet {
+				obj.Spec.AccessMode = v1alpha1.AccessMode(defaultPodAccessMode)
+			}
 			if err := r.Client.Create(context.Background(), obj); err != nil {
 				return err
 			}
@@ -228,11 +222,23 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 				return common.ResultRequeueAfter10sec, nil
 			}
 		}
+		nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName)
+		if !ncExist {
+			message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
+		if !r.VPCService.ValidateNetworkConfig(nc) {
+			// if network config is not valid, no need to retry, skip processing
+			message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns)
+			r.namespaceError(&ctx, obj, message, nil)
+			return common.ResultRequeueAfter10sec, nil
+		}
 
 		if _, err := r.createNetworkInfoCR(&ctx, obj, ns, ncName); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
-		if err := r.createDefaultSubnetSet(ns); err != nil {
+		if err := r.createDefaultSubnetSet(ns, nc.DefaultPodSubnetAccessMode); err != nil {
 			return common.ResultRequeueAfter10sec, nil
 		}
 		return common.ResultNormal, nil

pkg/controllers/networkinfo/vpcnetworkconfig_handler.go

@@ -121,17 +121,17 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*comm
 	}
 
 	ninfo := &commontypes.VPCNetworkConfigInfo{
-		IsDefault:               isDefaultNetworkConfigCR(vpcConfigCR),
-		Org:                     org,
-		Name:                    vpcConfigCR.Name,
-		DefaultGatewayPath:      vpcConfigCR.Spec.DefaultGatewayPath,
-		EdgeClusterPath:         vpcConfigCR.Spec.EdgeClusterPath,
-		NsxtProject:             project,
-		ExternalIPv4Blocks:      vpcConfigCR.Spec.ExternalIPv4Blocks,
-		PrivateIPv4CIDRs:        vpcConfigCR.Spec.PrivateIPv4CIDRs,
-		DefaultIPv4SubnetSize:   vpcConfigCR.Spec.DefaultIPv4SubnetSize,
-		DefaultSubnetAccessMode: vpcConfigCR.Spec.DefaultSubnetAccessMode,
-		ShortID:                 vpcConfigCR.Spec.ShortID,
+		IsDefault:                  isDefaultNetworkConfigCR(vpcConfigCR),
+		Org:                        org,
+		Name:                       vpcConfigCR.Name,
+		DefaultGatewayPath:         vpcConfigCR.Spec.DefaultGatewayPath,
+		EdgeClusterPath:            vpcConfigCR.Spec.EdgeClusterPath,
+		NsxtProject:                project,
+		ExternalIPv4Blocks:         vpcConfigCR.Spec.ExternalIPv4Blocks,
+		PrivateIPv4CIDRs:           vpcConfigCR.Spec.PrivateIPv4CIDRs,
+		DefaultIPv4SubnetSize:      vpcConfigCR.Spec.DefaultIPv4SubnetSize,
+		DefaultPodSubnetAccessMode: vpcConfigCR.Spec.DefaultPodSubnetAccessMode,
+		ShortID:                    vpcConfigCR.Spec.ShortID,
 	}
 	return ninfo, nil
 }

pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go

@@ -73,22 +73,22 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 	assert.NotNil(t, e)
 
 	spec1 := v1alpha1.VPCNetworkConfigurationSpec{
-		DefaultGatewayPath:      "test-gw-path-1",
-		EdgeClusterPath:         "test-edge-path-1",
-		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
-		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
-		DefaultIPv4SubnetSize:   64,
-		DefaultSubnetAccessMode: "Public",
-		NSXTProject:             "/orgs/default/projects/nsx_operator_e2e_test",
+		DefaultGatewayPath:         "test-gw-path-1",
+		EdgeClusterPath:            "test-edge-path-1",
+		ExternalIPv4Blocks:         []string{"external-ipb-1", "external-ipb-2"},
+		PrivateIPv4CIDRs:           []string{"private-ipb-1", "private-ipb-2"},
+		DefaultIPv4SubnetSize:      64,
+		DefaultPodSubnetAccessMode: "Public",
+		NSXTProject:                "/orgs/default/projects/nsx_operator_e2e_test",
 	}
 	spec2 := v1alpha1.VPCNetworkConfigurationSpec{
-		DefaultGatewayPath:      "test-gw-path-2",
-		EdgeClusterPath:         "test-edge-path-2",
-		ExternalIPv4Blocks:      []string{"external-ipb-1", "external-ipb-2"},
-		PrivateIPv4CIDRs:        []string{"private-ipb-1", "private-ipb-2"},
-		DefaultIPv4SubnetSize:   32,
-		DefaultSubnetAccessMode: "Private",
-		NSXTProject:             "/orgs/anotherOrg/projects/anotherProject",
+		DefaultGatewayPath:         "test-gw-path-2",
+		EdgeClusterPath:            "test-edge-path-2",
+		ExternalIPv4Blocks:         []string{"external-ipb-1", "external-ipb-2"},
+		PrivateIPv4CIDRs:           []string{"private-ipb-1", "private-ipb-2"},
+		DefaultIPv4SubnetSize:      32,
+		DefaultPodSubnetAccessMode: "Private",
+		NSXTProject:                "/orgs/anotherOrg/projects/anotherProject",
 	}
 	testCRD1 := v1alpha1.VPCNetworkConfiguration{
 		Spec: spec1,
@@ -133,7 +133,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 			assert.Equal(t, tt.org, nc.Org)
 			assert.Equal(t, tt.project, nc.NsxtProject)
 			assert.Equal(t, tt.subnetSize, nc.DefaultIPv4SubnetSize)
-			assert.Equal(t, tt.accessMode, nc.DefaultSubnetAccessMode)
+			assert.Equal(t, tt.accessMode, nc.DefaultPodSubnetAccessMode)
 			assert.Equal(t, tt.isDefault, nc.IsDefault)
 		})
 	}

pkg/controllers/subnet/subnet_controller.go

@@ -80,7 +80,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 				return ResultRequeue, err
 			}
 			if obj.Spec.AccessMode == "" {
-				obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+				obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 			}
 			if obj.Spec.IPv4SubnetSize == 0 {
 				obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize

pkg/controllers/subnetset/subnetset_controller.go

@@ -71,7 +71,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 					return ResultRequeue, err
 				}
 				if obj.Spec.AccessMode == "" {
-					obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode)
+					obj.Spec.AccessMode = v1alpha1.AccessMode("Private")
 				}
 				if obj.Spec.IPv4SubnetSize == 0 {
 					obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize