add readiness probe

Currently, manager pods turn Ready as soon as their binary starts.
However, it takes ~40 more seconds for the webhook to actually start
serving requests. In order to avoid that, we introduce a readiness
probe that marks the pod as Ready only once the server starts.

Please note that there is no way to do it properly with the current
controller-runtime. We don't have a hook point that would get executed
once the webhook has started. Therefore we have a Sleep there. That
should be dropped once
kubernetes-sigs/cluster-api#1855
is implemented.

Signed-off-by: Petr Horacek <phoracek@redhat.com>

config/default/manager/manager.yaml

@@ -92,6 +92,13 @@ spec:
         - containerPort: 8000
           name: webhook-server
           protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - cat
+            - /tmp/ready
+          initialDelaySeconds: 5
+          periodSeconds: 5
       terminationGracePeriodSeconds: 5
 ---
 apiVersion: policy/v1beta1

pkg/manager/manager.go

@@ -149,6 +149,17 @@ func (k *KubeMacPoolManager) Run(rangeStart, rangeEnd net.HardwareAddr) error {
 			return fmt.Errorf("unable to register webhooks to the manager error %v", err)
 		}
 
+		go func() {
+			// TODO: We cannot set the probe any closer to the server start.
+			// Therefore, we add this terrible sleep. It should be replaced by
+			// proper implementation once https://github.com/kubernetes-sigs/cluster-api/issues/1855
+			// is finished.
+			time.Sleep(time.Second * 20)
+			log.Info("Manager has hopefully started, marking as ready")
+			os.OpenFile("/tmp/ready", os.O_RDONLY|os.O_CREATE, 0666)
+		}()
+
+		log.Info("Starting manager")
 		err = mgr.Start(k.restartChannel)
 		if err != nil {
 			return fmt.Errorf("unable to run the manager error %v", err)