✨ (go/v3-alpha) Add the liveness and readiness probe in the manager d…

…eployment

pkg/plugins/golang/v3/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go

@@ -67,6 +67,7 @@ spec:
 {{- if not .ComponentConfig }}
       - name: manager
         args:
+        - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
 {{- end }}

pkg/plugins/golang/v3/scaffolds/internal/templates/config/manager/config.go

@@ -81,6 +81,18 @@ spec:
         name: manager
         securityContext:
           allowPrivilegeEscalation: false
+        livenessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m

pkg/plugins/golang/v3/scaffolds/internal/templates/config/manager/controller_manager_config.go

@@ -46,6 +46,8 @@ func (f *ControllerManagerConfig) SetTemplateDefaults() error {
 
 const controllerManagerConfigTemplate = `apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
 kind: ControllerManagerConfig
+health:
+  healthProbeBindAddress: :8081
 metrics:
   bindAddress: 127.0.0.1:8080
 webhook:

pkg/plugins/golang/v3/scaffolds/internal/templates/main.go

@@ -199,6 +199,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	%s
 )
 
@@ -217,7 +218,9 @@ func main() {
 {{- if not .ComponentConfig }}
 	var metricsAddr string
 	var enableLeaderElection bool
+	var probeAddr string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. " +
 		"Enabling this will ensure there is only one active controller manager.")
@@ -238,11 +241,12 @@ func main() {
 
 {{ if not .ComponentConfig }}
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		Port:               9443,
-		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   "{{ hashFNV .Repo }}.{{ .Domain }}",
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		Port:                   9443,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "{{ hashFNV .Repo }}.{{ .Domain }}",
 	})
 {{- else }}
 	var err error
@@ -264,6 +268,15 @@ func main() {
 
 	%s
 
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

test/e2e/v3/plugin_cluster_test.go

@@ -93,6 +93,8 @@ var _ = Describe("kubebuilder", func() {
 
 				GenerateV3(kbc, "v1")
 				Run(kbc)
+				RunHealthCheck(kbc)
+
 			})
 			It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() {
 				// Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field.
@@ -102,6 +104,8 @@ var _ = Describe("kubebuilder", func() {
 
 				GenerateV3(kbc, "v1beta1")
 				Run(kbc)
+				RunHealthCheck(kbc)
+
 			})
 		})
 	})
@@ -132,18 +136,7 @@ func Run(kbc *utils.TestContext) {
 	By("validating that the controller-manager pod is running as expected")
 	verifyControllerUp := func() error {
 		// Get pod name
-		podOutput, err := kbc.Kubectl.Get(
-			true,
-			"pods", "-l", "control-plane=controller-manager",
-			"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
-				"{{ \"\\n\" }}{{ end }}{{ end }}")
-		ExpectWithOffset(2, err).NotTo(HaveOccurred())
-		podNames := utils.GetNonEmptyLines(podOutput)
-		if len(podNames) != 1 {
-			return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
-		}
-		controllerPodName = podNames[0]
-		ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))
+		controllerPodName = getControllerPodName(kbc)
 
 		// Validate pod status
 		status, err := kbc.Kubectl.Get(
@@ -164,7 +157,7 @@ func Run(kbc *utils.TestContext) {
 		fmt.Sprintf("--serviceaccount=%s:default", kbc.Kubectl.Namespace))
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
-	_ = curlMetrics(kbc)
+	_ = curlEndpoint(kbc, "metrics")
 
 	By("validating that cert-manager has provisioned the certificate Secret")
 	EventuallyWithOffset(1, func() error {
@@ -238,7 +231,7 @@ func Run(kbc *utils.TestContext) {
 	}, time.Minute, time.Second).Should(Succeed())
 
 	By("validating that the created resource object gets reconciled in the controller")
-	metricsOutput := curlMetrics(kbc)
+	metricsOutput := curlEndpoint(kbc, "metrics")
 	ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf(
 		`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
 		strings.ToLower(kbc.Kind),
@@ -255,22 +248,51 @@ func Run(kbc *utils.TestContext) {
 	ExpectWithOffset(1, count).To(BeNumerically("==", 5))
 }
 
-// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned.
-func curlMetrics(kbc *utils.TestContext) string {
+func RunHealthCheck(kbc *utils.TestContext) {
+	By("validating that the healhz endpoint is accessible")
+	curlEndpoint(kbc, "healthz")
+
+	By("validating that the readyz endpoint is accessible")
+	curlEndpoint(kbc, "readyz")
+}
+
+// curlEndpoint curl's the /<endpoint>(/metrics or /healthz or /readyz) endpoint based on the input,
+// returning all logs once a 200 status is returned.
+func curlEndpoint(kbc *utils.TestContext, endpoint string) string {
+	var endpointPath string
+	var statusCode string
 	By("reading the metrics token")
 	b64Token, err := kbc.Kubectl.Get(true, "secrets", "-o=jsonpath={.items[0].data.token}")
 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
 	token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token))
 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
 	ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0))
 
+	if endpoint == "metrics" {
+		endpointPath = fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics",
+			kbc.TestSuffix, kbc.TestSuffix)
+		statusCode = "< HTTP/2 200"
+	} else {
+		//get controller pod name
+		controllerPodName := getControllerPodName(kbc)
+
+		controllerPodIP, err := kbc.Kubectl.Get(
+			true,
+			"pods", controllerPodName, "-o", "jsonpath={.status.podIP}")
+		ExpectWithOffset(2, err).NotTo(HaveOccurred())
+		if controllerPodIP == "" {
+			Fail("controller pod IP is not present")
+		}
+
+		endpointPath = fmt.Sprintf("http://%v:8081/%v", controllerPodIP, endpoint)
+		statusCode = "< HTTP/1.1 200"
+	}
 	By("creating a curl pod")
 	cmdOpts := []string{
 		"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
-		"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
-		fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics",
-			kbc.TestSuffix, kbc.TestSuffix),
+		"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token), endpointPath,
 	}
+
 	_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
 
@@ -289,17 +311,34 @@ func curlMetrics(kbc *utils.TestContext) string {
 	EventuallyWithOffset(2, verifyCurlUp, 30*time.Second, time.Second).Should(Succeed())
 
 	By("validating that the metrics endpoint is serving as expected")
-	var metricsOutput string
+	var endpointOutput string
 	getCurlLogs := func() string {
-		metricsOutput, err = kbc.Kubectl.Logs("curl")
+		endpointOutput, err = kbc.Kubectl.Logs("curl")
 		ExpectWithOffset(3, err).NotTo(HaveOccurred())
-		return metricsOutput
+		return endpointOutput
 	}
-	EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200"))
+	EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring(statusCode))
 
 	By("cleaning up the curl pod")
 	_, err = kbc.Kubectl.Delete(true, "pods/curl")
 	ExpectWithOffset(3, err).NotTo(HaveOccurred())
 
-	return metricsOutput
+	return endpointOutput
+}
+
+func getControllerPodName(kbc *utils.TestContext) string {
+	podOutput, err := kbc.Kubectl.Get(
+		true,
+		"pods", "-l", "control-plane=controller-manager",
+		"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
+			"{{ \"\\n\" }}{{ end }}{{ end }}")
+	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+	podNames := utils.GetNonEmptyLines(podOutput)
+	if len(podNames) != 1 {
+		Fail("expect 1 controller pods running, but got %d", len(podNames))
+	}
+	controllerPodName := podNames[0]
+	ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))
+
+	return controllerPodName
 }

testdata/project-v3-addon/config/default/manager_auth_proxy_patch.yaml

@@ -21,5 +21,6 @@ spec:
           name: https
       - name: manager
         args:
+        - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"

testdata/project-v3-addon/config/manager/controller_manager_config.yaml

@@ -1,5 +1,7 @@
 apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
 kind: ControllerManagerConfig
+health:
+  healthProbeBindAddress: :8081
 metrics:
   bindAddress: 127.0.0.1:8080
 webhook:

testdata/project-v3-addon/config/manager/manager.yaml

@@ -33,6 +33,18 @@ spec:
         name: manager
         securityContext:
           allowPrivilegeEscalation: false
+        livenessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m

testdata/project-v3-addon/main.go

@@ -28,6 +28,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	crewv1 "sigs.k8s.io/kubebuilder/testdata/project-v3-addon/api/v1"
@@ -50,7 +51,9 @@ func init() {
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
+	var probeAddr string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -63,11 +66,12 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		Port:               9443,
-		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   "52ea9610.testproject.org",
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		Port:                   9443,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "52ea9610.testproject.org",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -100,6 +104,15 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

testdata/project-v3-config/config/manager/controller_manager_config.yaml

@@ -1,5 +1,7 @@
 apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
 kind: ControllerManagerConfig
+health:
+  healthProbeBindAddress: :8081
 metrics:
   bindAddress: 127.0.0.1:8080
 webhook:

testdata/project-v3-config/config/manager/manager.yaml

@@ -31,6 +31,18 @@ spec:
         name: manager
         securityContext:
           allowPrivilegeEscalation: false
+        livenessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m

testdata/project-v3-config/main.go

@@ -28,6 +28,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	crewv1 "sigs.k8s.io/kubebuilder/testdata/project-v3-config/api/v1"
@@ -123,6 +124,15 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

testdata/project-v3-multigroup/config/default/manager_auth_proxy_patch.yaml

@@ -21,5 +21,6 @@ spec:
           name: https
       - name: manager
         args:
+        - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"

testdata/project-v3-multigroup/config/manager/controller_manager_config.yaml

@@ -1,5 +1,7 @@
 apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
 kind: ControllerManagerConfig
+health:
+  healthProbeBindAddress: :8081
 metrics:
   bindAddress: 127.0.0.1:8080
 webhook: