Add liveness and readiness probes

This adds liveness and readiness probes to the HNC Deployment using functionality recently added to controller-runtime. The probes configuration might need some adjustments of timeouts etc. to work without issues for a typical HNC deployment. The initial configuration is what the most recent version of kubebuilder suggests when scaffolding a new controller.

Testing: Ran all tests successfully on my local workstation, but to adjust the timeouts etc. this should be tested in a live cluster.

cmd/manager/main.go

@@ -24,6 +24,7 @@ import (
 	"contrib.go.opencensus.io/exporter/stackdriver"
 	"github.com/go-logr/zapr"
 	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
 	// Change to use v1 when we only need to support 1.17 and higher kubernetes versions.
 	stdzap "go.uber.org/zap"
@@ -51,6 +52,7 @@ var (
 )
 
 var (
+	probeAddr               string
 	metricsAddr             string
 	enableStackdriver       bool
 	maxReconciles           int
@@ -112,6 +114,7 @@ func main() {
 
 func parseFlags() {
 	setupLog.Info("Parsing flags")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableStackdriver, "enable-stackdriver", true, "If true, export metrics to stackdriver")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
@@ -217,16 +220,27 @@ func createManager() ctrl.Manager {
 	// it turns out to be harmful.
 	cfg.Burst = int(cfg.QPS * 1.5)
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   leaderElectionId,
-		Port:               webhookServerPort,
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       leaderElectionId,
+		Port:                   webhookServerPort,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create manager")
 		os.Exit(1)
 	}
+
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	return mgr
 }
 

config/manager/manager.yaml

@@ -50,6 +50,18 @@ spec:
         - "--excluded-namespace=kube-node-lease"
         image: controller:latest
         name: manager
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m