Enable HNC leader election in controllers

cmd/manager/main.go

@@ -19,6 +19,7 @@ import (
 	"flag"
 	"os"
 	"strings"
+	"time"
 
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"contrib.go.opencensus.io/exporter/stackdriver"
@@ -169,11 +170,20 @@ func main() {
 	// TODO: Better understand the behaviour of Burst, and consider making it equal to QPS if
 	// it turns out to be harmful.
 	cfg.Burst = int(cfg.QPS * 1.5)
+
+	// Update leader election leases config, to more sensible defaults
+	leaseDuration := 90 * time.Second // default is 15s
+	renewDeadline := 60 * time.Second // default is 10s
+	retryPeriod := 10 * time.Second   // default is 2s
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		LeaderElectionID:   leaderElectionId,
+		LeaseDuration:      &leaseDuration,
+		RenewDeadline:      &renewDeadline,
+		RetryPeriod:        &retryPeriod,
 		Port:               webhookServerPort,
 	})
 	if err != nil {
@@ -199,6 +209,7 @@ func main() {
 }
 
 func startControllers(mgr ctrl.Manager, certsCreated chan struct{}) {
+
 	// The controllers won't work until the webhooks are operating, and those won't work until the
 	// certs are all in place.
 	setupLog.Info("Waiting for certificate generation to complete")

config/rbac/role.yaml

@@ -42,3 +42,12 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update

internal/forest/forest.go

@@ -48,6 +48,9 @@ type TypeSyncer interface {
 	// reconciler if necessary.
 	SetMode(context.Context, logr.Logger, api.SynchronizationMode) error
 
+	// MakeLeader make reconciler a leader, enabling write operations
+	MakeLeader(context.Context, logr.Logger) error
+
 	// GetMode gets the propagation mode of objects that are handled by the reconciler who implements the interface.
 	GetMode() api.SynchronizationMode
 

internal/reconcilers/anchor.go

@@ -49,6 +49,9 @@ type AnchorReconciler struct {
 	// https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches.html) that is used to
 	// enqueue additional objects that need updating.
 	Affected chan event.GenericEvent
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 // Reconcile sets up some basic variables and then calls the business logic. It currently
@@ -73,6 +76,11 @@ func (r *AnchorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, err
 	}
 
+	// If not leader exit early, without performing any write operations
+	if !r.Leader {
+		return ctrl.Result{}, nil
+	}
+
 	// Always delete anchor (and any other HNC CRs) in the excluded namespaces and
 	// early exit.
 	if config.ExcludedNamespaces[pnm] {

internal/reconcilers/hierarchy_config.go

@@ -77,6 +77,9 @@ type HierarchyConfigReconciler struct {
 
 	// sar is the Subnamespace Anchor Reconciler
 	sar *AnchorReconciler
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 // +kubebuilder:rbac:groups=hnc.x-k8s.io,resources=hierarchies,verbs=get;list;watch;create;update;patch;delete
@@ -171,6 +174,11 @@ func (r *HierarchyConfigReconciler) reconcile(ctx context.Context, log logr.Logg
 	// Sync the Hierarchy singleton with the in-memory forest.
 	needUpdateObjects := r.syncWithForest(log, nsInst, inst, deletingCRD, anms)
 
+	// If not leader exit early, without performing any write operations
+	if !r.Leader {
+		return nil
+	}
+
 	// Write back if anything's changed. Early-exit if we just write back exactly what we had and this
 	// isn't the first time we're syncing.
 	updated, err := r.writeInstances(ctx, log, origHC, inst, origNS, nsInst)

internal/reconcilers/hnc_config.go

@@ -65,6 +65,9 @@ type ConfigReconciler struct {
 	// it's just a boolean (nil or non-nil), everything else is just for logging.
 	enqueueReasons     map[string]int
 	enqueueReasonsLock sync.Mutex
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom write operations
+	Leader bool
 }
 
 type gvkMode struct {
@@ -108,6 +111,17 @@ func (r *ConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	// Create or sync corresponding ObjectReconcilers, if needed.
 	syncErr := r.syncObjectReconcilers(ctx, inst)
 
+	// Update ObjectReconcilers if they are from HNC leader instance
+	if r.Leader {
+		for _, gvkMode := range r.activeGVKMode {
+			if ts := r.Forest.GetTypeSyncer(gvkMode.gvk); ts != nil {
+				if err := ts.MakeLeader(ctx, r.Log); err != nil {
+					r.Log.Error(err, "Couldn't make leader")
+				}
+			}
+		}
+	}
+
 	// Set the status for each type.
 	r.setTypeStatuses(inst)
 
@@ -369,6 +383,7 @@ func (r *ConfigReconciler) createObjectReconciler(gvk schema.GroupVersionKind, m
 		Affected:          make(chan event.GenericEvent),
 		AffectedNamespace: r.HierarchyConfigUpdates,
 		propagatedObjects: namespacedNameSet{},
+		Leader:            r.Leader,
 	}
 
 	// TODO: figure out MaxConcurrentReconciles option - https://github.com/kubernetes-sigs/hierarchical-namespaces/issues/291

internal/reconcilers/object.go

@@ -95,6 +95,9 @@ type ObjectReconciler struct {
 
 	// propagatedObjects contains all propagated objects of the GVK handled by this reconciler.
 	propagatedObjects namespacedNameSet
+
+	// Leader denotes whether this reconciler is from HNC leader and can perfrom wrote actions
+	Leader bool
 }
 
 // HNC doesn't actually need all these permissions, but we *do* need to have them to be able to
@@ -169,6 +172,20 @@ func (r *ObjectReconciler) SetMode(ctx context.Context, log logr.Logger, mode ap
 	return nil
 }
 
+// MakeLeader makes the object reconciler a leader, enabling write operations
+func (r *ObjectReconciler) MakeLeader(ctx context.Context, log logr.Logger) error {
+	log = log.WithValues("gvk", r.GVK)
+	if !r.Leader {
+		log.Info("Changing HA mode of the object reconciler to become leader")
+		r.Leader = true
+		err := r.enqueueAllObjects(ctx, r.Log)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // GetNumPropagatedObjects returns the number of propagated objects of the GVK handled by this object reconciler.
 func (r *ObjectReconciler) GetNumPropagatedObjects() int {
 	r.propagatedObjectsLock.Lock()
@@ -219,7 +236,13 @@ func (r *ObjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	// Sync with the forest and perform any required actions.
 	actions, srcInst := r.syncWithForest(ctx, log, inst)
-	return resp, r.operate(ctx, log, actions, inst, srcInst)
+
+	// Only complete on actions if reconciler is from Leader instance
+	if !r.Leader {
+		return resp, nil
+	}
+	err := r.operate(ctx, log, actions, inst, srcInst)
+	return resp, err
 }
 
 // syncWithForest syncs the object instance with the in-memory forest. It returns the action to take on

internal/reconcilers/setup.go

@@ -95,6 +95,14 @@ func Create(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, useFakeClient
 		return fmt.Errorf("cannot create Config reconciler: %s", err.Error())
 	}
 
+	// Watch for leader election to enable controller wries
+	go func() {
+		<-mgr.Elected()
+		sar.Leader = true
+		hcr.Leader = true
+		hnccrSingleton.Leader = true
+	}()
+
 	return nil
 }