Add support for VPC mixed mode

build/yaml/webhook/manifests.yaml

@@ -30,3 +30,36 @@ webhooks:
     resources:
     - subnetsets
   sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: subnetset
+      namespace: vmware-system-nsx
+      path: /validate-vpc-enablement
+  failurePolicy: Fail
+  name: vpcnetwork.validating.nsx.vmware.com
+  rules:
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+      resources:
+        - networkinfos
+        - nsxserviceaccount
+        - securitypolicies
+        - staticroutes
+        - subnetports
+        - subnets
+        - subnetsets
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha2
+      operations:
+        - CREATE
+      resources:
+        - ippools
+  sideEffects: None

cmd/main.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
@@ -35,6 +36,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
@@ -133,12 +135,13 @@ func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
 	}
 }
 
-func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider) {
+func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
 	nsReconciler := &namespacecontroller.NamespaceReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		NSXConfig:  cf,
-		VPCService: vpcService,
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		NSXConfig:       cf,
+		VPCService:      vpcService,
+		NetworkProvider: networkProvider,
 	}
 
 	if err := nsReconciler.Start(mgr); err != nil {
@@ -149,14 +152,26 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp
 
 func main() {
 	log.Info("starting NSX Operator")
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	mgrOptions := ctrl.Options{
 		Scheme:                  scheme,
 		HealthProbeBindAddress:  config.ProbeAddr,
 		Metrics:                 metricsserver.Options{BindAddress: config.MetricsAddr},
 		LeaderElection:          cf.HAEnabled(),
 		LeaderElectionNamespace: nsxOperatorNamespace,
 		LeaderElectionID:        "nsx-operator",
-	})
+	}
+
+	enableWebhook := true
+	if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
+		log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
+		enableWebhook = false
+	} else {
+		mgrOptions.WebhookServer = webhook.NewServer(webhook.Options{
+			Port:    config.WebhookServerPort,
+			CertDir: config.WebhookCertDir,
+		})
+	}
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
 	if err != nil {
 		log.Error(err, "failed to init manager")
 		os.Exit(1)
@@ -181,6 +196,10 @@ func main() {
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork {
+		if !enableWebhook {
+			log.Error(nil, "Webhook cert is not provided, can't filter out the CRs in a non-VPC namespace")
+			os.Exit(1)
+		}
 		// Check NSX version for VPC networking mode
 		if !commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
 			log.Error(nil, "VPC mode cannot be enabled if NSX version is lower than 4.1.1")
@@ -218,28 +237,27 @@ func main() {
 			os.Exit(1)
 		}
 		// Start controllers which only supports VPC
+		var vpcNetworkProvider vpcnetwork.VPCNetworkProvider
+		if cf.CoeConfig.EnableVPCMixedMode {
+			vpcNetworkProvider = vpcnetwork.StartNetworkController(mgr)
+		}
 		StartNetworkInfoController(mgr, vpcService)
-		StartNamespaceController(mgr, cf, vpcService)
+		StartNamespaceController(mgr, cf, vpcService, vpcNetworkProvider)
 		// Start subnet/subnetset controller.
 		if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {
 			os.Exit(1)
 		}
-		enableWebhook := true
-		if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
-			log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
-			enableWebhook = false
-		}
 		if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil {
 			os.Exit(1)
 		}
 
 		node.StartNodeController(mgr, nodeService)
 		staticroutecontroller.StartStaticRouteController(mgr, staticRouteService)
 		subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
-		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
+		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService, vpcNetworkProvider)
 		StartIPPoolController(mgr, ipPoolService, vpcService)
-		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
-		service.StartServiceLbController(mgr, commonService)
+		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService, vpcNetworkProvider)
+		service.StartServiceLbController(mgr, commonService, vpcNetworkProvider)
 	}
 	// Start controllers which can run in non-VPC mode
 	securitypolicycontroller.StartSecurityPolicyController(mgr, commonService, vpcService)

go.mod

@@ -6,14 +6,15 @@ replace (
 	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
-        github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
 )
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/deckarep/golang-set v1.8.0
 	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/zapr v1.2.4
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -23,6 +24,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.4
+	github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240305035435-c992c623aad3
 	github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f
 	github.com/vmware-tanzu/vm-operator/api v1.8.2
@@ -53,7 +55,6 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gibson042/canonicaljson-go v1.0.3 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect

go.sum

@@ -177,8 +177,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1 h1:r8RuSJnLEStOdGTfaZeOjH4rvRB4Gm/N1+qtU16VrI0=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo=
 github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
 github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
 github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=

pkg/config/config.go

@@ -94,8 +94,9 @@ type DefaultConfig struct {
 }
 
 type CoeConfig struct {
-	Cluster          string `ini:"cluster"`
-	EnableVPCNetwork bool   `ini:"enable_vpc_network"`
+	Cluster            string `ini:"cluster"`
+	EnableVPCNetwork   bool   `ini:"enable_vpc_network"`
+	EnableVPCMixedMode bool   `ini:"enable_vpc_mixed_mode"`
 }
 
 type NsxConfig struct {
@@ -238,6 +239,10 @@ func (operatorConfig *NSXOperatorConfig) validate() error {
 	if err := operatorConfig.NsxConfig.validate(operatorConfig.CoeConfig.EnableVPCNetwork); err != nil {
 		return err
 	}
+	if operatorConfig.CoeConfig.EnableVPCMixedMode && !operatorConfig.CoeConfig.EnableVPCNetwork {
+		configLog.Error("VPC mixed mode is enabled but VPC is not enabled")
+		return errors.New("VPC mixed mode can't be enabled without VPC network enablement")
+	}
 	// TODO, verify if user&pwd, cert, jwt has any of them provided
 	return nil
 }

pkg/controllers/namespace/namespace_controller.go

@@ -8,17 +8,21 @@ import (
 	"errors"
 	"fmt"
 
+	netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -35,10 +39,11 @@ var (
 // Using vpcservice provider instead of vpc service to prevent
 // invoking method that should be exposed to other module.
 type NamespaceReconciler struct {
-	Client     client.Client
-	Scheme     *apimachineryruntime.Scheme
-	NSXConfig  *config.NSXOperatorConfig
-	VPCService types.VPCServiceProvider
+	Client          client.Client
+	Scheme          *apimachineryruntime.Scheme
+	NSXConfig       *config.NSXOperatorConfig
+	VPCService      types.VPCServiceProvider
+	NetworkProvider vpcnetwork.VPCNetworkProvider
 }
 
 func (r *NamespaceReconciler) getDefaultNetworkConfigName() (string, error) {
@@ -179,6 +184,13 @@ func (r *NamespaceReconciler) insertNamespaceNetworkconfigBinding(ns string, ann
 	return nil
 }
 
+func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	if r.NetworkProvider != nil {
+		return r.NetworkProvider.ReconcileWithVPCFilters("namespace", ctx, req, r.reconcile)
+	}
+	return r.reconcile(ctx, req)
+}
+
 /*
 	VPC creation strategy:
 
@@ -193,7 +205,7 @@ We suppose namespace should have following annotations:
   - If the ns do not have either of the annotation above, then we believe it is using default VPC, try to search
     default VPC in network config CR store. The default VPC network config CR's name is "default".
 */
-func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *NamespaceReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	obj := &v1.Namespace{}
 	log.Info("reconciling K8s namespace", "namespace", req.NamespacedName)
 	metrics.CounterInc(r.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNamespace)
@@ -251,6 +263,14 @@ func (r *NamespaceReconciler) setupWithManager(mgr ctrl.Manager) error {
 			controller.Options{
 				MaxConcurrentReconciles: common.NumReconcile(),
 			}).
+		Watches(
+			&netopv1alpha1.Network{},
+			&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: func(namespace string) ([]apitypes.NamespacedName, error) {
+				obj := apitypes.NamespacedName{Name: namespace, Namespace: namespace}
+				return []apitypes.NamespacedName{obj}, nil
+			}},
+			builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
+		).
 		Complete(r)
 }
 

pkg/controllers/namespace/namespace_controller_test.go

@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	vpcnetworktesting "github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork/testing"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
@@ -40,9 +41,10 @@ func createNameSpaceReconciler() *NamespaceReconciler {
 	}
 
 	return &NamespaceReconciler{
-		Client:     fake.NewClientBuilder().Build(),
-		Scheme:     fake.NewClientBuilder().Build().Scheme(),
-		VPCService: service,
+		Client:          fake.NewClientBuilder().Build(),
+		Scheme:          fake.NewClientBuilder().Build().Scheme(),
+		VPCService:      service,
+		NetworkProvider: &vpcnetworktesting.FakeVPCNetworkProvider{},
 	}
 }