Add liveness and startup/readiness probes

cmd/manager/main.go

@@ -51,6 +51,7 @@ var (
 )
 
 var (
+	probeAddr               string
 	metricsAddr             string
 	enableStackdriver       bool
 	maxReconciles           int
@@ -89,6 +90,7 @@ func main() {
 	metricsCleanupFn := enableMetrics()
 	defer metricsCleanupFn()
 	mgr := createManager()
+	setupChecks(mgr)
 
 	// Make sure certs are generated and valid if webhooks are enabled and internal certs are used.
 	setupLog.Info("Starting certificate generation")
@@ -112,6 +114,7 @@ func main() {
 
 func parseFlags() {
 	setupLog.Info("Parsing flags")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableStackdriver, "enable-stackdriver", true, "If true, export metrics to stackdriver")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
@@ -217,11 +220,12 @@ func createManager() ctrl.Manager {
 	// it turns out to be harmful.
 	cfg.Burst = int(cfg.QPS * 1.5)
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   leaderElectionId,
-		Port:               webhookServerPort,
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       leaderElectionId,
+		Port:                   webhookServerPort,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create manager")
@@ -230,6 +234,17 @@ func createManager() ctrl.Manager {
 	return mgr
 }
 
+func setupChecks(mgr ctrl.Manager) {
+	if err := mgr.AddHealthzCheck("healthz", mgr.GetWebhookServer().StartedChecker()); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", mgr.GetWebhookServer().StartedChecker()); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+}
+
 func startControllers(mgr ctrl.Manager, certsReady chan struct{}) {
 	// The controllers won't work until the webhooks are operating, and those won't work until the
 	// certs are all in place.

config/manager/manager.yaml

@@ -50,6 +50,18 @@ spec:
         - "--excluded-namespace=kube-node-lease"
         image: controller:latest
         name: manager
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          failureThreshold: 1
+          periodSeconds: 10
+        startupProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          failureThreshold: 100
+          periodSeconds: 5
         resources:
           limits:
             cpu: 100m