Add support for VPC mixed mode

1. Add controller for CR Network to maintain the default Network resource in system Namespaces. 2. Add watch in Pod/Namepsace/Service/NetworkPolicy controller to requeue resources when Network type is switch to VPC. 3. Add precheck on if the default Network type is VPC or not in resource create events. 4. Add webhook in NCP owned resource creations to check if the default Network type is VPC or not in a Namespace.
vmware-tanzu · Jun 24, 2024 · 4ff5fa9 · 4ff5fa9
1 parent 566bc19
commit 4ff5fa9
Show file tree

Hide file tree

Showing 18 changed files with 1,410 additions and 75 deletions.
diff --git a/build/yaml/webhook/manifests.yaml b/build/yaml/webhook/manifests.yaml
@@ -30,3 +30,36 @@ webhooks:
     resources:
     - subnetsets
   sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: subnetset
+      namespace: vmware-system-nsx
+      path: /validate-vpc-enablement
+  failurePolicy: Fail
+  name: vpcnetwork.validating.nsx.vmware.com
+  rules:
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+      resources:
+        - networkinfos
+        - nsxserviceaccount
+        - securitypolicies
+        - staticroutes
+        - subnetports
+        - subnets
+        - subnetsets
+    - apiGroups:
+        - nsx.vmware.com
+      apiVersions:
+        - v1alpha2
+      operations:
+        - CREATE
+      resources:
+        - ippools
+  sideEffects: None
diff --git a/cmd/main.go b/cmd/main.go
@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2"
@@ -35,6 +36,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
@@ -133,12 +135,13 @@ func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
 	}
 }
 
-func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider) {
+func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vpcService common.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
 	nsReconciler := &namespacecontroller.NamespaceReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		NSXConfig:  cf,
-		VPCService: vpcService,
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		NSXConfig:       cf,
+		VPCService:      vpcService,
+		NetworkProvider: networkProvider,
 	}
 
 	if err := nsReconciler.Start(mgr); err != nil {
@@ -149,14 +152,26 @@ func StartNamespaceController(mgr ctrl.Manager, cf *config.NSXOperatorConfig, vp
 
 func main() {
 	log.Info("starting NSX Operator")
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	mgrOptions := ctrl.Options{
 		Scheme:                  scheme,
 		HealthProbeBindAddress:  config.ProbeAddr,
 		Metrics:                 metricsserver.Options{BindAddress: config.MetricsAddr},
 		LeaderElection:          cf.HAEnabled(),
 		LeaderElectionNamespace: nsxOperatorNamespace,
 		LeaderElectionID:        "nsx-operator",
-	})
+	}
+
+	enableWebhook := true
+	if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
+		log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
+		enableWebhook = false
+	} else {
+		mgrOptions.WebhookServer = webhook.NewServer(webhook.Options{
+			Port:    config.WebhookServerPort,
+			CertDir: config.WebhookCertDir,
+		})
+	}
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
 	if err != nil {
 		log.Error(err, "failed to init manager")
 		os.Exit(1)
@@ -181,6 +196,10 @@ func main() {
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork {
+		if !enableWebhook {
+			log.Error(nil, "Webhook cert is not provided, can't filter out the CRs in a non-VPC namespace")
+			os.Exit(1)
+		}
 		// Check NSX version for VPC networking mode
 		if !commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
 			log.Error(nil, "VPC mode cannot be enabled if NSX version is lower than 4.1.1")
@@ -218,28 +237,24 @@ func main() {
 			os.Exit(1)
 		}
 		// Start controllers which only supports VPC
+		vpcNetworkProvider := vpcnetwork.StartNetworkController(mgr)
 		StartNetworkInfoController(mgr, vpcService)
-		StartNamespaceController(mgr, cf, vpcService)
+		StartNamespaceController(mgr, cf, vpcService, vpcNetworkProvider)
 		// Start subnet/subnetset controller.
 		if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {
 			os.Exit(1)
 		}
-		enableWebhook := true
-		if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) {
-			log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir)
-			enableWebhook = false
-		}
 		if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil {
 			os.Exit(1)
 		}
 
 		node.StartNodeController(mgr, nodeService)
 		staticroutecontroller.StartStaticRouteController(mgr, staticRouteService)
 		subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService)
-		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService)
+		pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService, vpcNetworkProvider)
 		StartIPPoolController(mgr, ipPoolService, vpcService)
-		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService)
-		service.StartServiceLbController(mgr, commonService)
+		networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService, vpcNetworkProvider)
+		service.StartServiceLbController(mgr, commonService, vpcNetworkProvider)
 	}
 	// Start controllers which can run in non-VPC mode
 	securitypolicycontroller.StartSecurityPolicyController(mgr, commonService, vpcService)

diff --git a/go.mod b/go.mod
@@ -6,14 +6,15 @@ replace (
 	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1 => ./pkg/apis/v1alpha1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2 => ./pkg/apis/v1alpha2
-        github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
+	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
 )
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/deckarep/golang-set v1.8.0
 	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/zapr v1.2.4
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -23,6 +24,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.4
+	github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1
 	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240305035435-c992c623aad3
 	github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f
 	github.com/vmware-tanzu/vm-operator/api v1.8.2
@@ -53,7 +55,6 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gibson042/canonicaljson-go v1.0.3 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect

diff --git a/go.sum b/go.sum
@@ -177,8 +177,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f h1:EV4eiUQr3QpUGfTtqdVph0+bmE+3cj0aNJpd9n2qTdo=
-github.com/vmware-tanzu/nsx-operator/pkg/client v0.0.0-20240102061654-537b080e159f/go.mod h1:dzob8tUzpAREQPtbbjQs4b1UyQDR37B2TiIdg8WJSRM=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1 h1:r8RuSJnLEStOdGTfaZeOjH4rvRB4Gm/N1+qtU16VrI0=
+github.com/vmware-tanzu/net-operator-api v0.0.0-20240529180459-ccac5a20bda1/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo=
 github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4=
 github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54=
 github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=

diff --git a/pkg/controllers/namespace/namespace_controller.go b/pkg/controllers/namespace/namespace_controller.go
@@ -8,17 +8,21 @@ import (
 	"errors"
 	"fmt"
 
+	netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -35,10 +39,11 @@ var (
 // Using vpcservice provider instead of vpc service to prevent
 // invoking method that should be exposed to other module.
 type NamespaceReconciler struct {
-	Client     client.Client
-	Scheme     *apimachineryruntime.Scheme
-	NSXConfig  *config.NSXOperatorConfig
-	VPCService types.VPCServiceProvider
+	Client          client.Client
+	Scheme          *apimachineryruntime.Scheme
+	NSXConfig       *config.NSXOperatorConfig
+	VPCService      types.VPCServiceProvider
+	NetworkProvider vpcnetwork.VPCNetworkProvider
 }
 
 func (r *NamespaceReconciler) getDefaultNetworkConfigName() (string, error) {
@@ -179,6 +184,10 @@ func (r *NamespaceReconciler) insertNamespaceNetworkconfigBinding(ns string, ann
 	return nil
 }
 
+func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	return r.NetworkProvider.ReconcileWithVPCFilters("namespace", ctx, req, r.reconcile)
+}
+
 /*
 	VPC creation strategy:
 
@@ -193,7 +202,7 @@ We suppose namespace should have following annotations:
   - If the ns do not have either of the annotation above, then we believe it is using default VPC, try to search
     default VPC in network config CR store. The default VPC network config CR's name is "default".
 */
-func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *NamespaceReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	obj := &v1.Namespace{}
 	log.Info("reconciling K8s namespace", "namespace", req.NamespacedName)
 	metrics.CounterInc(r.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNamespace)
@@ -251,6 +260,14 @@ func (r *NamespaceReconciler) setupWithManager(mgr ctrl.Manager) error {
 			controller.Options{
 				MaxConcurrentReconciles: common.NumReconcile(),
 			}).
+		Watches(
+			&netopv1alpha1.Network{},
+			&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: func(namespace string) ([]apitypes.NamespacedName, error) {
+				obj := apitypes.NamespacedName{Name: namespace, Namespace: namespace}
+				return []apitypes.NamespacedName{obj}, nil
+			}},
+			builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
+		).
 		Complete(r)
 }
 

diff --git a/pkg/controllers/namespace/namespace_controller_test.go b/pkg/controllers/namespace/namespace_controller_test.go
@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	vpcnetworktesting "github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork/testing"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
@@ -40,9 +41,10 @@ func createNameSpaceReconciler() *NamespaceReconciler {
 	}
 
 	return &NamespaceReconciler{
-		Client:     fake.NewClientBuilder().Build(),
-		Scheme:     fake.NewClientBuilder().Build().Scheme(),
-		VPCService: service,
+		Client:          fake.NewClientBuilder().Build(),
+		Scheme:          fake.NewClientBuilder().Build().Scheme(),
+		VPCService:      service,
+		NetworkProvider: &vpcnetworktesting.FakeVPCNetworkProvider{},
 	}
 }
 

diff --git a/pkg/controllers/networkpolicy/networkpolicy_controller.go b/pkg/controllers/networkpolicy/networkpolicy_controller.go
@@ -11,18 +11,21 @@ import (
 	"sync"
 	"time"
 
+	"github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/vpcnetwork"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
@@ -42,10 +45,11 @@ var (
 
 // NetworkPolicyReconciler reconciles a NetworkPolicy object
 type NetworkPolicyReconciler struct {
-	Client   client.Client
-	Scheme   *apimachineryruntime.Scheme
-	Service  *securitypolicy.SecurityPolicyService
-	Recorder record.EventRecorder
+	Client          client.Client
+	Scheme          *apimachineryruntime.Scheme
+	Service         *securitypolicy.SecurityPolicyService
+	Recorder        record.EventRecorder
+	NetworkProvider vpcnetwork.VPCNetworkProvider
 }
 
 func updateFail(r *NetworkPolicyReconciler, c *context.Context, o *networkingv1.NetworkPolicy, e *error) {
@@ -71,6 +75,10 @@ func deleteSuccess(r *NetworkPolicyReconciler, _ *context.Context, o *networking
 func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	// Use once.Do to ensure gc is called only once
 	once.Do(func() { go r.GarbageCollector(make(chan bool), servicecommon.GCInterval) })
+	return r.NetworkProvider.ReconcileWithVPCFilters("networkpolicy", ctx, req, r.reconcile)
+}
+
+func (r *NetworkPolicyReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	networkPolicy := &networkingv1.NetworkPolicy{}
 	log.Info("reconciling networkpolicy", "networkpolicy", req.NamespacedName)
 	metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType)
@@ -135,6 +143,11 @@ func (r *NetworkPolicyReconciler) setupWithManager(mgr ctrl.Manager) error {
 			controller.Options{
 				MaxConcurrentReconciles: common.NumReconcile(),
 			}).
+		Watches(
+			&v1alpha1.Network{},
+			&vpcnetwork.EnqueueRequestForNetwork{Client: r.Client, Lister: r.listNetworkPolicies},
+			builder.WithPredicates(vpcnetwork.PredicateFuncsByNetwork),
+		).
 		Complete(r)
 }
 
@@ -191,11 +204,28 @@ func (r *NetworkPolicyReconciler) GarbageCollector(cancel chan bool, timeout tim
 	}
 }
 
-func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider) {
+func (r *NetworkPolicyReconciler) listNetworkPolicies(ns string) ([]types.NamespacedName, error) {
+	npList := &networkingv1.NetworkPolicyList{}
+	err := r.Client.List(context.Background(), npList, client.InNamespace(ns))
+	if err != nil {
+		return nil, err
+	}
+	nsNames := make([]types.NamespacedName, 0)
+	for _, np := range npList.Items {
+		nsNames = append(nsNames, types.NamespacedName{
+			Namespace: np.Namespace,
+			Name:      np.Name,
+		})
+	}
+	return nsNames, nil
+}
+
+func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider, networkProvider vpcnetwork.VPCNetworkProvider) {
 	networkPolicyReconcile := NetworkPolicyReconciler{
-		Client:   mgr.GetClient(),
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor("networkpolicy-controller"),
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Recorder:        mgr.GetEventRecorderFor("networkpolicy-controller"),
+		NetworkProvider: networkProvider,
 	}
 	networkPolicyReconcile.Service = securitypolicy.GetSecurityService(commonService, vpcService)
 	if err := networkPolicyReconcile.Start(mgr); err != nil {