Enable HNC leader election in controllers

cmd/manager/main.go

@@ -19,6 +19,7 @@ import (
 	"flag"
 	"os"
 	"strings"
+	"time"
 
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"contrib.go.opencensus.io/exporter/stackdriver"
@@ -167,11 +168,24 @@ func main() {
 	// TODO: Better understand the behaviour of Burst, and consider making it equal to QPS if
 	// it turns out to be harmful.
 	cfg.Burst = int(cfg.QPS * 1.5)
+
+	// If leader election is disabled then treat this instance as the HNC leader
+	if !enableLeaderElection {
+		config.IsLeader = true
+	}
+	// Update leader election leases config, to more sensible defaults
+	leaseDuration := 90 * time.Second // default is 15s
+	renewDeadline := 60 * time.Second // default is 10s
+	retryPeriod := 10 * time.Second   // default is 2s
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		LeaderElectionID:   leaderElectionId,
+		LeaseDuration:      &leaseDuration,
+		RenewDeadline:      &renewDeadline,
+		RetryPeriod:        &retryPeriod,
 		Port:               webhookServerPort,
 	})
 	if err != nil {
@@ -197,6 +211,7 @@ func main() {
 }
 
 func startControllers(mgr ctrl.Manager, certsCreated chan struct{}) {
+
 	// The controllers won't work until the webhooks are operating, and those won't work until the
 	// certs are all in place.
 	setupLog.Info("Waiting for certificate generation to complete")

config/rbac/role.yaml

@@ -42,3 +42,12 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update

internal/anchor/reconciler.go

@@ -56,6 +56,12 @@ type Reconciler struct {
 // Reconcile sets up some basic variables and then calls the business logic. It currently
 // only handles the creation of the namespaces but no deletion or state reporting yet.
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+
+	// If not leader exiting, without performing any write operations
+	if !config.IsLeader {
+		return ctrl.Result{}, nil
+	}
+
 	log := logutils.WithRID(r.Log).WithValues("trigger", req.NamespacedName)
 	log.V(1).Info("Reconciling anchor")
 

internal/config/default_config.go

@@ -8,3 +8,6 @@ package config
 // This value is controlled by the --unpropagated-annotation command line, which may be set multiple
 // times.
 var UnpropagatedAnnotations []string
+
+// IsLeader is global which repesents whether this HNC instance is the leader
+var IsLeader bool

internal/forest/forest.go

@@ -37,8 +37,8 @@ type namedNamespaces map[string]*Namespace
 // TypeSyncer syncs objects of a specific type. Reconcilers implement the interface so that they can be
 // called by the HierarchyReconciler if the hierarchy changes.
 type TypeSyncer interface {
-	// SyncNamespace syncs objects of a namespace for a specific type.
-	SyncNamespace(context.Context, logr.Logger, string) error
+	// SyncObjects syncs objects of a namespace for a specific type.
+	SyncObjects(context.Context, logr.Logger, string) error
 
 	// Provides the GVK that is handled by the reconciler who implements the interface.
 	GetGVK() schema.GroupVersionKind

internal/hierarchyconfig/reconciler.go

@@ -191,6 +191,11 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, nm string)
 	// Sync the Hierarchy singleton with the in-memory forest.
 	needUpdateObjects := r.syncWithForest(log, nsInst, inst, deletingCRD, anms)
 
+	// If not leader exit early, without performing any write operations
+	if !config.IsLeader {
+		return nil
+	}
+
 	// Write back if anything's changed. Early-exit if we just write back exactly what we had and this
 	// isn't the first time we're syncing.
 	updated, err := r.writeInstances(ctx, log, origHC, inst, origNS, nsInst)
@@ -679,7 +684,7 @@ func (r *Reconciler) updateObjects(ctx context.Context, log logr.Logger, ns stri
 	trs := r.Forest.GetTypeSyncers()
 	r.Forest.Unlock()
 	for _, tr := range trs {
-		if err := tr.SyncNamespace(ctx, log, ns); err != nil {
+		if err := tr.SyncObjects(ctx, log, ns); err != nil {
 			return err
 		}
 	}

internal/hncconfig/reconciler.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 
 	api "sigs.k8s.io/hierarchical-namespaces/api/v1alpha2"
+	"sigs.k8s.io/hierarchical-namespaces/internal/config"
 	"sigs.k8s.io/hierarchical-namespaces/internal/crd"
 	"sigs.k8s.io/hierarchical-namespaces/internal/forest"
 	"sigs.k8s.io/hierarchical-namespaces/internal/objects"
@@ -131,6 +132,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	// Load all conditions
 	r.loadNamespaceConditions(inst)
 
+	// Exit early of not the leader
+	if !config.IsLeader {
+		return ctrl.Result{}, nil
+	}
+
 	// Write back to the apiserver.
 	if err := r.writeSingleton(ctx, inst); err != nil {
 		r.Log.Error(err, "Couldn't write singleton")
@@ -631,3 +637,18 @@ func gvkForGR(gr schema.GroupResource, allRes []*restmapper.APIGroupResources) (
 	}
 	return schema.GroupVersionKind{}, &GVKErr{api.ReasonResourceNotFound, fmt.Sprintf("Resource %q not found", gr)}
 }
+
+// RequeueAll calls SyncObjects on all type reconcillers (required when instance becomes leader)
+func (r *Reconciler) RequeueAll() {
+	r.Log.V(1).Info("Requeue all objects (hierarchy updated or new namespace found)")
+	// Use mutex to guard the read from the types list of the forest to prevent the ConfigReconciler
+	// from modifying the list at the same time.
+	r.Forest.Lock()
+	trs := r.Forest.GetTypeSyncers()
+	r.Forest.Unlock()
+	for _, tr := range trs {
+		if err := tr.SyncObjects(context.Background(), r.Log, ""); err != nil {
+			r.Log.V(1).Error(err, "Failed to SyncObjects for HNCConfig")
+		}
+	}
+}

internal/objects/reconciler.go

@@ -97,6 +97,9 @@ type Reconciler struct {
 
 	// propagatedObjects contains all propagated objects of the GVK handled by this reconciler.
 	propagatedObjects namespacedNameSet
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom wrote actions
+	Leader bool
 }
 
 type HNCConfigReconcilerType interface {
@@ -109,12 +112,19 @@ type HNCConfigReconcilerType interface {
 //
 // +kubebuilder:rbac:groups=*,resources=*,verbs=*
 
-// SyncNamespace can be called manually by the HierarchyConfigReconciler when the hierarchy changes.
+// SyncObjects can be called manually by the HierarchyConfigReconciler when the hierarchy changes.
 // It enqueues all the current objects in the namespace and local copies of the original objects
 // in the ancestors.
-func (r *Reconciler) SyncNamespace(ctx context.Context, log logr.Logger, ns string) error {
+func (r *Reconciler) SyncObjects(ctx context.Context, log logr.Logger, ns string) error {
 	log = log.WithValues("gvk", r.GVK)
 
+	// If ns is empty, then enqueue all objects (used when HNC instances become leader)
+	if ns == "" {
+		if err := r.enqueueAllObjects(ctx, log); err != nil {
+			return err
+		}
+	}
+
 	// Enqueue all the current objects in the namespace because some of them may have been deleted.
 	if err := r.enqueueLocalObjects(ctx, log, ns); err != nil {
 		return err
@@ -225,7 +235,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	// Sync with the forest and perform any required actions.
 	actions, srcInst := r.syncWithForest(ctx, log, inst)
-	return resp, r.operate(ctx, log, actions, inst, srcInst)
+
+	// Only complete on actions if reconciler is from Leader instance
+	if !r.Leader {
+		return resp, nil
+	}
+	err := r.operate(ctx, log, actions, inst, srcInst)
+	return resp, err
 }
 
 // syncWithForest syncs the object instance with the in-memory forest. It returns the action to take on
@@ -513,9 +529,9 @@ func (r *Reconciler) enqueueLocalObjects(ctx context.Context, log logr.Logger, n
 	return nil
 }
 
-// enqueuePropagatedObjects is only called from SyncNamespace. It's the only place a forest lock is
-// needed in SyncNamespace, so we made it into a function with forest lock instead of holding the
-// lock for the entire SyncNamespace.
+// enqueuePropagatedObjects is only called from SyncObjects. It's the only place a forest lock is
+// needed in SyncObjects, so we made it into a function with forest lock instead of holding the
+// lock for the entire SyncObjects.
 func (r *Reconciler) enqueuePropagatedObjects(ctx context.Context, log logr.Logger, ns string) {
 	r.Forest.Lock()
 	defer r.Forest.Unlock()

internal/setup/reconcilers.go

@@ -7,6 +7,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 
 	"sigs.k8s.io/hierarchical-namespaces/internal/anchor"
+	"sigs.k8s.io/hierarchical-namespaces/internal/config"
 	"sigs.k8s.io/hierarchical-namespaces/internal/crd"
 	"sigs.k8s.io/hierarchical-namespaces/internal/forest"
 	"sigs.k8s.io/hierarchical-namespaces/internal/hierarchyconfig"
@@ -59,5 +60,14 @@ func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, us
 		return fmt.Errorf("cannot create Hierarchy reconciler: %s", err.Error())
 	}
 
+	// If LeaderElection is enabled then Watch for Elected() to enable controller writes
+	if !config.IsLeader {
+		go func() {
+			<-mgr.Elected()
+			config.IsLeader = true
+			hnccfgr.RequeueAll()
+		}()
+	}
+
 	return nil
 }