Watch for node create event in NicClusterPolicy reconcile loop

We need to watch for node create event for correct cluster scale up
by adding 'network.nvidia.com/operator.mofed.wait' label.

Signed-off-by: Ivan Kolodiazhny <ikolodiazhny@nvidia.com>

controllers/nicclusterpolicy_controller.go

@@ -28,9 +28,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
@@ -236,6 +240,25 @@ func (r *NicClusterPolicyReconciler) handleUnsupportedInstance(instance *mellano
 	return err
 }
 
+func createPredicate() predicate.Predicate {
+	return predicate.Funcs{
+		CreateFunc: func(event.CreateEvent) bool {
+			return true
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			return false
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			// Evaluates to false if the object has been confirmed deleted.
+			return false
+		},
+		GenericFunc: func(e event.GenericEvent) bool {
+			// Evaluates to false if the object has been confirmed deleted.
+			return false
+		},
+	}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 //
 //nolint:dupl
@@ -249,20 +272,35 @@ func (r *NicClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	}
 	r.stateManager = stateManager
 
-	builder := ctrl.NewControllerManagedBy(mgr).
+	controller := ctrl.NewControllerManagedBy(mgr).
 		For(&mellanoxv1alpha1.NicClusterPolicy{}).
 		// Watch for changes to primary resource NicClusterPolicy
 		Watches(&source.Kind{Type: &mellanoxv1alpha1.NicClusterPolicy{}}, &handler.EnqueueRequestForObject{})
 
-	// Watch for changes to secondary resource DaemonSet and requeue the owner NicClusterPolicy
+	// we always add object with a same(static) key to the queue to reduce
+	// reconciliation count
+	qHandler := func(q workqueue.RateLimitingInterface) {
+		q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
+			Namespace: "nic-cluster-policy-reconcile-namespace",
+			Name:      "nic-cluster-policy-reconcile-name",
+		}})
+	}
+
+	createEnqueue := handler.Funcs{
+		CreateFunc: func(e event.CreateEvent, q workqueue.RateLimitingInterface) {
+			qHandler(q)
+		},
+	}
+	nodePredicates := builder.WithPredicates(createPredicate())
+
 	ws := stateManager.GetWatchSources()
 	r.Log.V(consts.LogLevelInfo).Info("Watch Sources", "Kind:", ws)
 	for i := range ws {
-		builder = builder.Watches(ws[i], &handler.EnqueueRequestForOwner{
+		controller = controller.Watches(ws[i], &handler.EnqueueRequestForOwner{
 			IsController: true,
 			OwnerType:    &mellanoxv1alpha1.NicClusterPolicy{},
-		})
+		}).Watches(&source.Kind{Type: &corev1.Node{}}, createEnqueue, nodePredicates)
 	}
 
-	return builder.Complete(r)
+	return controller.Complete(r)
 }