diff --git a/build/yaml/crd/nsx.vmware.com_ippools.yaml b/build/yaml/crd/nsx.vmware.com_ippools.yaml index e6cf505c4..3402a97f7 100644 --- a/build/yaml/crd/nsx.vmware.com_ippools.yaml +++ b/build/yaml/crd/nsx.vmware.com_ippools.yaml @@ -172,10 +172,12 @@ spec: type: object type: array type: - description: Type defines the type of this IPPool, Public or Private. + description: Type defines the type of this IPPool, Public, Private + or Project. enum: - Public - Private + - Project type: string type: object status: diff --git a/build/yaml/crd/nsx.vmware.com_subnets.yaml b/build/yaml/crd/nsx.vmware.com_subnets.yaml index 7d5de517b..ad5d9f002 100644 --- a/build/yaml/crd/nsx.vmware.com_subnets.yaml +++ b/build/yaml/crd/nsx.vmware.com_subnets.yaml @@ -85,6 +85,8 @@ spec: enum: - Private - Public + - Project + - Isolated type: string advancedConfig: description: Subnet advanced configuration. diff --git a/build/yaml/crd/nsx.vmware.com_subnetsets.yaml b/build/yaml/crd/nsx.vmware.com_subnetsets.yaml index aac120f98..46ebb269c 100644 --- a/build/yaml/crd/nsx.vmware.com_subnetsets.yaml +++ b/build/yaml/crd/nsx.vmware.com_subnetsets.yaml @@ -85,6 +85,8 @@ spec: enum: - Private - Public + - Project + - Isolated type: string advancedConfig: description: Subnet advanced configuration. diff --git a/build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml b/build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml index 91e3bf8fa..daa0d839c 100644 --- a/build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml +++ b/build/yaml/crd/nsx.vmware.com_vpcnetworkconfigurations.yaml @@ -53,6 +53,15 @@ spec: When a field is not set in a Namespace's VPCNetworkConfiguration, the Namespace will use the value in the default VPCNetworkConfiguration. properties: + defaultAccessModeForPod: + description: DefaultAccessModeForPod defines the access mode of the + default SubnetSet for PodVM. Must be Public or Private. + enum: + - Public + - Private + - Project + - Isolated + type: string defaultGatewayPath: description: PolicyPath of Tier0 or Tier0 VRF gateway. type: string @@ -61,13 +70,6 @@ spec: description: Default size of Subnet based upon estimated workload count. Defaults to 26. type: integer - defaultSubnetAccessMode: - description: DefaultSubnetAccessMode defines the access mode of the - default SubnetSet for PodVM and VM. Must be Public or Private. - enum: - - Public - - Private - type: string edgeClusterPath: description: Edge cluster path on which the networking elements will be created. diff --git a/go.mod b/go.mod index a5b825376..80ebbb971 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,11 @@ module github.com/vmware-tanzu/nsx-operator go 1.21 +replace ( + github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis + github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client +) + require ( github.com/agiledragon/gomonkey/v2 v2.9.0 github.com/apparentlymart/go-cidr v1.1.0 diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go b/pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go index 5473a8021..54a36375d 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/subnet_types.go @@ -16,7 +16,7 @@ type SubnetSpec struct { // +kubebuilder:validation:Minimum:=16 IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` // Access mode of Subnet, accessible only from within VPC or from outside VPC. - // +kubebuilder:validation:Enum=Private;Public + // +kubebuilder:validation:Enum=Private;Public;Project;Isolated AccessMode AccessMode `json:"accessMode,omitempty"` // Subnet CIDRS. // +kubebuilder:validation:MinItems=0 diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go b/pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go index 5c6864893..cb18ceae3 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/subnetset_types.go @@ -14,7 +14,7 @@ type SubnetSetSpec struct { // +kubebuilder:validation:Minimum:=16 IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` // Access mode of Subnet, accessible only from within VPC or from outside VPC. - // +kubebuilder:validation:Enum=Private;Public + // +kubebuilder:validation:Enum=Private;Public;Project;Isolated AccessMode AccessMode `json:"accessMode,omitempty"` // Subnet advanced configuration. AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"` diff --git a/pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go b/pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go index c22205e31..b6ca0eb40 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha1/vpcnetworkconfiguration_types.go @@ -37,10 +37,15 @@ type VPCNetworkConfigurationSpec struct { // Defaults to 26. // +kubebuilder:default=26 DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"` - // DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM. + // DefaultAccessModeForPod defines the access mode of the default SubnetSet for PodVM. // Must be Public or Private. - // +kubebuilder:validation:Enum=Public;Private - DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"` + // +kubebuilder:validation:Enum=Public;Private;Project;Isolated + DefaultAccessModeForPod string `json:"defaultAccessModeForPod,omitempty"` + // ShortID specifies Identifier to use when displaying VPC context in logs. + // Less than equal to 8 characters. + // +kubebuilder:validation:MaxLength=8 + // +optional + ShortID string `json:"shortID,omitempty"` } // VPCNetworkConfigurationStatus defines the observed state of VPCNetworkConfiguration diff --git a/pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go b/pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go index 469dc43ef..ef1a32cce 100644 --- a/pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go +++ b/pkg/apis/nsx.vmware.com/v1alpha2/ippool_types.go @@ -36,8 +36,8 @@ type IPPoolList struct { // IPPoolSpec defines the desired state of IPPool. type IPPoolSpec struct { - // Type defines the type of this IPPool, Public or Private. - // +kubebuilder:validation:Enum=Public;Private + // Type defines the type of this IPPool, Public, Private or Project. + // +kubebuilder:validation:Enum=Public;Private;Project // +optional Type string `json:"type,omitempty"` // Subnets defines set of subnets need to be allocated. diff --git a/pkg/apis/v1alpha1/subnet_types.go b/pkg/apis/v1alpha1/subnet_types.go index 5473a8021..54a36375d 100644 --- a/pkg/apis/v1alpha1/subnet_types.go +++ b/pkg/apis/v1alpha1/subnet_types.go @@ -16,7 +16,7 @@ type SubnetSpec struct { // +kubebuilder:validation:Minimum:=16 IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` // Access mode of Subnet, accessible only from within VPC or from outside VPC. - // +kubebuilder:validation:Enum=Private;Public + // +kubebuilder:validation:Enum=Private;Public;Project;Isolated AccessMode AccessMode `json:"accessMode,omitempty"` // Subnet CIDRS. // +kubebuilder:validation:MinItems=0 diff --git a/pkg/apis/v1alpha1/subnetset_types.go b/pkg/apis/v1alpha1/subnetset_types.go index 5c6864893..cb18ceae3 100644 --- a/pkg/apis/v1alpha1/subnetset_types.go +++ b/pkg/apis/v1alpha1/subnetset_types.go @@ -14,7 +14,7 @@ type SubnetSetSpec struct { // +kubebuilder:validation:Minimum:=16 IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` // Access mode of Subnet, accessible only from within VPC or from outside VPC. - // +kubebuilder:validation:Enum=Private;Public + // +kubebuilder:validation:Enum=Private;Public;Project;Isolated AccessMode AccessMode `json:"accessMode,omitempty"` // Subnet advanced configuration. AdvancedConfig AdvancedConfig `json:"advancedConfig,omitempty"` diff --git a/pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go b/pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go index 168752f28..b6ca0eb40 100644 --- a/pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go +++ b/pkg/apis/v1alpha1/vpcnetworkconfiguration_types.go @@ -37,10 +37,10 @@ type VPCNetworkConfigurationSpec struct { // Defaults to 26. // +kubebuilder:default=26 DefaultIPv4SubnetSize int `json:"defaultIPv4SubnetSize,omitempty"` - // DefaultSubnetAccessMode defines the access mode of the default SubnetSet for PodVM and VM. + // DefaultAccessModeForPod defines the access mode of the default SubnetSet for PodVM. // Must be Public or Private. - // +kubebuilder:validation:Enum=Public;Private - DefaultSubnetAccessMode string `json:"defaultSubnetAccessMode,omitempty"` + // +kubebuilder:validation:Enum=Public;Private;Project;Isolated + DefaultAccessModeForPod string `json:"defaultAccessModeForPod,omitempty"` // ShortID specifies Identifier to use when displaying VPC context in logs. // Less than equal to 8 characters. // +kubebuilder:validation:MaxLength=8 diff --git a/pkg/apis/v1alpha2/ippool_types.go b/pkg/apis/v1alpha2/ippool_types.go index 469dc43ef..ef1a32cce 100644 --- a/pkg/apis/v1alpha2/ippool_types.go +++ b/pkg/apis/v1alpha2/ippool_types.go @@ -36,8 +36,8 @@ type IPPoolList struct { // IPPoolSpec defines the desired state of IPPool. type IPPoolSpec struct { - // Type defines the type of this IPPool, Public or Private. - // +kubebuilder:validation:Enum=Public;Private + // Type defines the type of this IPPool, Public, Private or Project. + // +kubebuilder:validation:Enum=Public;Private;Project // +optional Type string `json:"type,omitempty"` // Subnets defines set of subnets need to be allocated. diff --git a/pkg/controllers/ippool/ippool_controller.go b/pkg/controllers/ippool/ippool_controller.go index a5d0b52e8..92cdd3d47 100644 --- a/pkg/controllers/ippool/ippool_controller.go +++ b/pkg/controllers/ippool/ippool_controller.go @@ -141,7 +141,7 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr updateFail(r, &ctx, obj, &err) return resultRequeue, err } - obj.Spec.Type = vpcNetworkConfig.DefaultSubnetAccessMode + obj.Spec.Type = "Private" } if obj.ObjectMeta.DeletionTimestamp.IsZero() { diff --git a/pkg/controllers/namespace/namespace_controller.go b/pkg/controllers/namespace/namespace_controller.go index ce33b661d..ee3cb1f92 100644 --- a/pkg/controllers/namespace/namespace_controller.go +++ b/pkg/controllers/namespace/namespace_controller.go @@ -60,18 +60,6 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec log.Info("vpc cr already exist, skip creating", "VPC", vpcs.Items[0].Name) return &vpcs.Items[0], nil } - nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName) - if !ncExist { - message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns) - r.namespaceError(ctx, obj, message, nil) - return nil, errors.New(message) - } - if !r.VPCService.ValidateNetworkConfig(nc) { - // if network config is not valid, no need to retry, skip processing - message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns) - r.namespaceError(ctx, obj, message, nil) - return nil, errors.New(message) - } // create vpc cr with existing vpc network config vpcCR := BuildVPCCR(ns, ncName, vpcName) @@ -91,7 +79,7 @@ func (r *NamespaceReconciler) createVPCCR(ctx *context.Context, obj client.Objec return vpcCR, nil } -func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error { +func (r *NamespaceReconciler) createDefaultSubnetSet(ns string, defaultPodAccessMode string) error { defaultSubnetSets := map[string]string{ types.DefaultVMSubnetSet: types.LabelDefaultVMSubnetSet, types.DefaultPodSubnetSet: types.LabelDefaultPodSubnetSet, @@ -127,6 +115,12 @@ func (r *NamespaceReconciler) createDefaultSubnetSet(ns string) error { }, }, } + if name == types.DefaultVMSubnetSet { + // use "Private" type for VM + obj.Spec.AccessMode = v1alpha1.AccessMode("Private") + } else if name == types.DefaultPodSubnetSet { + obj.Spec.AccessMode = v1alpha1.AccessMode(defaultPodAccessMode) + } if err := r.Client.Create(context.Background(), obj); err != nil { return err } @@ -246,11 +240,23 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return common.ResultRequeueAfter10sec, nil } } + nc, ncExist := r.VPCService.GetVPCNetworkConfig(ncName) + if !ncExist { + message := fmt.Sprintf("missing network config %s for namespace %s", ncName, ns) + r.namespaceError(&ctx, obj, message, nil) + return common.ResultRequeueAfter10sec, nil + } + if !r.VPCService.ValidateNetworkConfig(nc) { + // if network config is not valid, no need to retry, skip processing + message := fmt.Sprintf("invalid network config %s for namespace %s, missing private cidr", ncName, ns) + r.namespaceError(&ctx, obj, message, nil) + return common.ResultRequeueAfter10sec, nil + } if _, err := r.createVPCCR(&ctx, obj, ns, ncName, createVpcName); err != nil { return common.ResultRequeueAfter10sec, nil } - if err := r.createDefaultSubnetSet(ns); err != nil { + if err := r.createDefaultSubnetSet(ns, nc.DefaultAccessModeForPod); err != nil { return common.ResultRequeueAfter10sec, nil } return common.ResultNormal, nil diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 5b170f43e..cbb9fd304 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -80,7 +80,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ResultRequeue, err } if obj.Spec.AccessMode == "" { - obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode) + obj.Spec.AccessMode = v1alpha1.AccessMode("Private") } if obj.Spec.IPv4SubnetSize == 0 { obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 78c027d50..18d01ea2c 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -71,7 +71,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ResultRequeue, err } if obj.Spec.AccessMode == "" { - obj.Spec.AccessMode = v1alpha1.AccessMode(vpcNetworkConfig.DefaultSubnetAccessMode) + obj.Spec.AccessMode = v1alpha1.AccessMode("Private") } if obj.Spec.IPv4SubnetSize == 0 { obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize diff --git a/pkg/controllers/vpc/vpc_utils.go b/pkg/controllers/vpc/vpc_utils.go index 2b4bd1580..84d302321 100644 --- a/pkg/controllers/vpc/vpc_utils.go +++ b/pkg/controllers/vpc/vpc_utils.go @@ -162,7 +162,7 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*type ExternalIPv4Blocks: vpcConfigCR.Spec.ExternalIPv4Blocks, PrivateIPv4CIDRs: vpcConfigCR.Spec.PrivateIPv4CIDRs, DefaultIPv4SubnetSize: vpcConfigCR.Spec.DefaultIPv4SubnetSize, - DefaultSubnetAccessMode: vpcConfigCR.Spec.DefaultSubnetAccessMode, + DefaultAccessModeForPod: vpcConfigCR.Spec.DefaultAccessModeForPod, ShortID: vpcConfigCR.Spec.ShortID, } return ninfo, nil diff --git a/pkg/controllers/vpc/vpc_utils_test.go b/pkg/controllers/vpc/vpc_utils_test.go index 6ee365c23..8d5e58541 100644 --- a/pkg/controllers/vpc/vpc_utils_test.go +++ b/pkg/controllers/vpc/vpc_utils_test.go @@ -79,7 +79,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) { ExternalIPv4Blocks: []string{"external-ipb-1", "external-ipb-2"}, PrivateIPv4CIDRs: []string{"private-ipb-1", "private-ipb-2"}, DefaultIPv4SubnetSize: 64, - DefaultSubnetAccessMode: "Public", + DefaultAccessModeForPod: "Public", NSXTProject: "/orgs/default/projects/nsx_operator_e2e_test", } spec2 := v1alpha1.VPCNetworkConfigurationSpec{ @@ -88,7 +88,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) { ExternalIPv4Blocks: []string{"external-ipb-1", "external-ipb-2"}, PrivateIPv4CIDRs: []string{"private-ipb-1", "private-ipb-2"}, DefaultIPv4SubnetSize: 32, - DefaultSubnetAccessMode: "Private", + DefaultAccessModeForPod: "Private", NSXTProject: "/orgs/anotherOrg/projects/anotherProject", } testCRD1 := v1alpha1.VPCNetworkConfiguration{ @@ -134,7 +134,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) { assert.Equal(t, tt.org, nc.Org) assert.Equal(t, tt.project, nc.NsxtProject) assert.Equal(t, tt.subnetSize, nc.DefaultIPv4SubnetSize) - assert.Equal(t, tt.accessMode, nc.DefaultSubnetAccessMode) + assert.Equal(t, tt.accessMode, nc.DefaultAccessModeForPod) assert.Equal(t, tt.isDefault, nc.IsDefault) }) } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 74b4bc5be..e73b4ee8b 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -203,6 +203,6 @@ type VPCNetworkConfigInfo struct { ExternalIPv4Blocks []string PrivateIPv4CIDRs []string DefaultIPv4SubnetSize int - DefaultSubnetAccessMode string + DefaultAccessModeForPod string ShortID string } diff --git a/test/e2e/nsx_subnet_test.go b/test/e2e/nsx_subnet_test.go index 0023da2a3..4684083fd 100644 --- a/test/e2e/nsx_subnet_test.go +++ b/test/e2e/nsx_subnet_test.go @@ -27,7 +27,7 @@ const ( SubnetDeletionTimeout = 300 * time.Second ) -func verifySubnetSetCR(subnetSet string) bool { +func verifySubnetSetCR(subnetSet string, subnetType string) bool { vpcNetworkConfig, err := testData.crdClientset.NsxV1alpha1().VPCNetworkConfigurations().Get(context.TODO(), VPCNetworkConfigCRName, v1.GetOptions{}) if err != nil { log.Printf("Failed to get VPCNetworkConfiguration %s: %v", VPCNetworkConfigCRName, err) @@ -38,8 +38,14 @@ func verifySubnetSetCR(subnetSet string) bool { log.Printf("Failed to get %s/%s: %s", E2ENamespace, subnetSet, err) return false } - if string(subnetSetCR.Spec.AccessMode) != vpcNetworkConfig.Spec.DefaultSubnetAccessMode { - log.Printf("AccessMode is %s, while it's expected to be %s", subnetSetCR.Spec.AccessMode, vpcNetworkConfig.Spec.DefaultSubnetAccessMode) + var desiredSubnetType string + if subnetType == "" { + desiredSubnetType = vpcNetworkConfig.Spec.DefaultAccessModeForPod + } else { + desiredSubnetType = subnetType + } + if string(subnetSetCR.Spec.AccessMode) != desiredSubnetType { + log.Printf("AccessMode is %s, while it's expected to be %s", subnetSetCR.Spec.AccessMode, desiredSubnetType) return false } if subnetSetCR.Spec.IPv4SubnetSize != vpcNetworkConfig.Spec.DefaultIPv4SubnetSize { @@ -74,8 +80,8 @@ func defaultSubnetSet(t *testing.T) { assertNil(t, err) // 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig. - assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet)) - assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet)) + assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet, "Private")) + assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet, "")) portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_1.yaml") err = applyYAML(portPath, E2ENamespace) @@ -157,7 +163,7 @@ func userSubnetSet(t *testing.T) { assertNil(t, err) // 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig. - assertTrue(t, verifySubnetSetCR(UserSubnetSet)) + assertTrue(t, verifySubnetSetCR(UserSubnetSet, "Private")) portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_2.yaml") err = applyYAML(portPath, E2ENamespace) @@ -186,8 +192,8 @@ func sharedSubnetSet(t *testing.T) { assertNil(t, err) // 2. Check `Ipv4SubnetSize` and `AccessMode` should be same with related fields in VPCNetworkConfig. - assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet)) - assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet)) + assertTrue(t, verifySubnetSetCR(common.DefaultVMSubnetSet, "Private")) + assertTrue(t, verifySubnetSetCR(common.DefaultPodSubnetSet, "")) portPath, _ := filepath.Abs("./manifest/testSubnet/subnetport_3.yaml") err = applyYAML(portPath, E2ENamespaceShared)