Enable HNC leader election in controllers

cmd/manager/main.go

@@ -19,6 +19,7 @@ import (
 	"flag"
 	"os"
 	"strings"
+	"time"
 
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"contrib.go.opencensus.io/exporter/stackdriver"
@@ -167,11 +168,20 @@ func main() {
 	// TODO: Better understand the behaviour of Burst, and consider making it equal to QPS if
 	// it turns out to be harmful.
 	cfg.Burst = int(cfg.QPS * 1.5)
+
+	// Update leader election leases config, to more sensible defaults
+	leaseDuration := 90 * time.Second // default is 15s
+	renewDeadline := 60 * time.Second // default is 10s
+	retryPeriod := 10 * time.Second   // default is 2s
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		LeaderElectionID:   leaderElectionId,
+		LeaseDuration:      &leaseDuration,
+		RenewDeadline:      &renewDeadline,
+		RetryPeriod:        &retryPeriod,
 		Port:               webhookServerPort,
 	})
 	if err != nil {
@@ -197,6 +207,7 @@ func main() {
 }
 
 func startControllers(mgr ctrl.Manager, certsCreated chan struct{}) {
+
 	// The controllers won't work until the webhooks are operating, and those won't work until the
 	// certs are all in place.
 	setupLog.Info("Waiting for certificate generation to complete")

config/rbac/role.yaml

@@ -42,3 +42,12 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update

internal/anchor/reconciler.go

@@ -51,11 +51,20 @@ type Reconciler struct {
 	// https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches.html) that is used to
 	// enqueue additional objects that need updating.
 	Affected chan event.GenericEvent
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 // Reconcile sets up some basic variables and then calls the business logic. It currently
 // only handles the creation of the namespaces but no deletion or state reporting yet.
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+
+	// If not leader exiting, without performing any write operations
+	if !config.IsLeader {
+		return ctrl.Result{}, nil
+	}
+
 	log := logutils.WithRID(r.Log).WithValues("trigger", req.NamespacedName)
 	log.V(1).Info("Reconciling anchor")
 

internal/config/default_config.go

@@ -8,3 +8,6 @@ package config
 // This value is controlled by the --unpropagated-annotation command line, which may be set multiple
 // times.
 var UnpropagatedAnnotations []string
+
+// IsLeader is global which repesents whether this HNC instance is the leader
+var IsLeader bool

internal/forest/forest.go

@@ -37,8 +37,8 @@ type namedNamespaces map[string]*Namespace
 // TypeSyncer syncs objects of a specific type. Reconcilers implement the interface so that they can be
 // called by the HierarchyReconciler if the hierarchy changes.
 type TypeSyncer interface {
-	// SyncNamespace syncs objects of a namespace for a specific type.
-	SyncNamespace(context.Context, logr.Logger, string) error
+	// SyncObjects syncs objects of a namespace for a specific type.
+	SyncObjects(context.Context, logr.Logger, string) error
 
 	// Provides the GVK that is handled by the reconciler who implements the interface.
 	GetGVK() schema.GroupVersionKind

internal/hierarchyconfig/reconciler.go

@@ -79,6 +79,9 @@ type Reconciler struct {
 	// These are interfaces to the other reconcilers
 	AnchorReconciler    AnchorReconcilerType
 	HNCConfigReconciler HNCConfigReconcilerType
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 type AnchorReconcilerType interface {
@@ -191,6 +194,11 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, nm string)
 	// Sync the Hierarchy singleton with the in-memory forest.
 	needUpdateObjects := r.syncWithForest(log, nsInst, inst, deletingCRD, anms)
 
+	// If not leader exit early, without performing any write operations
+	if !config.IsLeader {
+		return nil
+	}
+
 	// Write back if anything's changed. Early-exit if we just write back exactly what we had and this
 	// isn't the first time we're syncing.
 	updated, err := r.writeInstances(ctx, log, origHC, inst, origNS, nsInst)
@@ -679,7 +687,7 @@ func (r *Reconciler) updateObjects(ctx context.Context, log logr.Logger, ns stri
 	trs := r.Forest.GetTypeSyncers()
 	r.Forest.Unlock()
 	for _, tr := range trs {
-		if err := tr.SyncNamespace(ctx, log, ns); err != nil {
+		if err := tr.SyncObjects(ctx, log, ns); err != nil {
 			return err
 		}
 	}

internal/hncconfig/reconciler.go

@@ -67,6 +67,9 @@ type Reconciler struct {
 	// it's just a boolean (nil or non-nil), everything else is just for logging.
 	enqueueReasons     map[string]int
 	enqueueReasonsLock sync.Mutex
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 type gvkMode struct {
@@ -131,6 +134,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	// Load all conditions
 	r.loadNamespaceConditions(inst)
 
+	// Exit early of not the leader
+	if !r.Leader {
+		return ctrl.Result{}, nil
+	}
+
 	// Write back to the apiserver.
 	if err := r.writeSingleton(ctx, inst); err != nil {
 		r.Log.Error(err, "Couldn't write singleton")
@@ -386,6 +394,7 @@ func (r *Reconciler) createObjectReconciler(gvk schema.GroupVersionKind, mode ap
 		Affected:            make(chan event.GenericEvent),
 		AffectedNamespace:   r.HierarchyConfigUpdates,
 		HNCConfigReconciler: r,
+		Leader:              r.Leader,
 	}
 
 	// TODO: figure out MaxConcurrentReconciles option - https://github.com/kubernetes-sigs/hierarchical-namespaces/issues/291
@@ -631,3 +640,25 @@ func gvkForGR(gr schema.GroupResource, allRes []*restmapper.APIGroupResources) (
 	}
 	return schema.GroupVersionKind{}, &GVKErr{api.ReasonResourceNotFound, fmt.Sprintf("Resource %q not found", gr)}
 }
+
+// RequeueAll make all object syncers requeue all objects
+func (r *Reconciler) RequeueAll(ctx context.Context, log logr.Logger) error {
+	log = log.WithValues("gvk", r.GVK)
+	log.Info("RequeueAll object reconcilers")
+	for _, gvkMode := range r.activeGVKMode {
+		if ts := r.Forest.GetTypeSyncer(gvkMode.gvk); ts != nil {
+			err := ts.(ctx, r.Log)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+
+	// TODO: remove this as calling MakeLeader from main reconciler setup
+	// Update ObjectReconcilers if they are from HNC leader instance
+	// if r.Leader {
+
+	// }

internal/objects/reconciler.go

@@ -97,6 +97,9 @@ type Reconciler struct {
 
 	// propagatedObjects contains all propagated objects of the GVK handled by this reconciler.
 	propagatedObjects namespacedNameSet
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom wrote actions
+	Leader bool
 }
 
 type HNCConfigReconcilerType interface {
@@ -109,12 +112,19 @@ type HNCConfigReconcilerType interface {
 //
 // +kubebuilder:rbac:groups=*,resources=*,verbs=*
 
-// SyncNamespace can be called manually by the HierarchyConfigReconciler when the hierarchy changes.
+// SyncObjects can be called manually by the HierarchyConfigReconciler when the hierarchy changes.
 // It enqueues all the current objects in the namespace and local copies of the original objects
 // in the ancestors.
-func (r *Reconciler) SyncNamespace(ctx context.Context, log logr.Logger, ns string) error {
+func (r *Reconciler) SyncObjects(ctx context.Context, log logr.Logger, ns string) error {
 	log = log.WithValues("gvk", r.GVK)
 
+	// If ns is empty, then enqueue all objects (used when HNC instances become leader)
+	if ns == "" {
+		if err := r.enqueueAllObjects(ctx, log); err != nil {
+			return err
+		}
+	}
+
 	// Enqueue all the current objects in the namespace because some of them may have been deleted.
 	if err := r.enqueueLocalObjects(ctx, log, ns); err != nil {
 		return err
@@ -225,7 +235,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	// Sync with the forest and perform any required actions.
 	actions, srcInst := r.syncWithForest(ctx, log, inst)
-	return resp, r.operate(ctx, log, actions, inst, srcInst)
+
+	// Only complete on actions if reconciler is from Leader instance
+	if !r.Leader {
+		return resp, nil
+	}
+	err := r.operate(ctx, log, actions, inst, srcInst)
+	return resp, err
 }
 
 // syncWithForest syncs the object instance with the in-memory forest. It returns the action to take on
@@ -513,9 +529,9 @@ func (r *Reconciler) enqueueLocalObjects(ctx context.Context, log logr.Logger, n
 	return nil
 }
 
-// enqueuePropagatedObjects is only called from SyncNamespace. It's the only place a forest lock is
-// needed in SyncNamespace, so we made it into a function with forest lock instead of holding the
-// lock for the entire SyncNamespace.
+// enqueuePropagatedObjects is only called from SyncObjects. It's the only place a forest lock is
+// needed in SyncObjects, so we made it into a function with forest lock instead of holding the
+// lock for the entire SyncObjects.
 func (r *Reconciler) enqueuePropagatedObjects(ctx context.Context, log logr.Logger, ns string) {
 	r.Forest.Lock()
 	defer r.Forest.Unlock()
@@ -791,3 +807,17 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, maxReconciles int) error
 		WithOptions(opts).
 		Complete(r)
 }
+
+// MakeLeader makes the object reconciler a leader, enabling write operations
+func (r *Reconciler) MakeLeader(ctx context.Context, log logr.Logger) error {
+	log = log.WithValues("gvk", r.GVK)
+	if !r.Leader {
+		log.Info("Changing HA mode of the object reconciler to become leader")
+		r.Leader = true
+		err := r.SyncObjects(ctx, r.Log, "").enqueueAllObjects(ctx, r.Log)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/setup/reconcilers.go

@@ -7,6 +7,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 
 	"sigs.k8s.io/hierarchical-namespaces/internal/anchor"
+	"sigs.k8s.io/hierarchical-namespaces/internal/config"
 	"sigs.k8s.io/hierarchical-namespaces/internal/crd"
 	"sigs.k8s.io/hierarchical-namespaces/internal/forest"
 	"sigs.k8s.io/hierarchical-namespaces/internal/hierarchyconfig"
@@ -59,5 +60,15 @@ func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, us
 		return fmt.Errorf("cannot create Hierarchy reconciler: %s", err.Error())
 	}
 
+	// Watch for leader election to enable controller wries
+	go func() {
+		<-mgr.Elected()
+		config.IsLeader = true
+		//ar.Leader = true
+		//hnccfgr.Leader = true
+		//hcr.Leader = true
+
+	}()
+
 	return nil
 }