From 1aee555271b226cc96e6615a0a4c13be921dfca1 Mon Sep 17 00:00:00 2001 From: Ramon Perez Date: Wed, 31 Jan 2024 16:07:23 +0100 Subject: [PATCH] Add support for startup probes --- designs/component-config.md | 4 +- ...ve-cluster-specific-code-out-of-manager.md | 3 + pkg/config/v1alpha1/types.go | 4 ++ pkg/healthz/doc.go | 4 +- pkg/manager/internal.go | 29 ++++++++ .../internal/integration/manager_test.go | 1 + pkg/manager/manager.go | 15 ++++ pkg/manager/manager_test.go | 68 ++++++++++++++++++- 8 files changed, 122 insertions(+), 6 deletions(-) diff --git a/designs/component-config.md b/designs/component-config.md index 8aebec4f96..7ab07b6814 100644 --- a/designs/component-config.md +++ b/designs/component-config.md @@ -1,7 +1,7 @@ # ComponentConfig Controller Runtime Support Author: @christopherhein -Last Updated on: 03/02/2020 +Last Updated on: 31/01/2024 ## Table of Contents @@ -91,6 +91,7 @@ type ManagerConfiguration interface { GetReadinessEndpointName() string GetLivenessEndpointName() string + GetStartupEndpointName() string GetPort() int GetHost() string @@ -161,6 +162,7 @@ type ControllerManagerConfigurationHealth struct { ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` LivenessEndpointName string `json:"livenessEndpointName,omitempty"` + StartupEndpointName string `json:"startupEndpointName,omitempty"` } ``` diff --git a/designs/move-cluster-specific-code-out-of-manager.md b/designs/move-cluster-specific-code-out-of-manager.md index 67b7a419a5..dbd38335a9 100644 --- a/designs/move-cluster-specific-code-out-of-manager.md +++ b/designs/move-cluster-specific-code-out-of-manager.md @@ -116,6 +116,9 @@ type Manager interface { // AddReadyzCheck allows you to add Readyz checker AddReadyzCheck(name string, check healthz.Checker) error + // AddStartzCheck allows you to add Startz checker + AddStartzCheck(name string, check healthz.Checker) error + // Start starts all registered Controllers and blocks until the Stop channel is closed. // Returns an error if there is an error starting any controller. // If LeaderElection is used, the binary must be exited immediately after this returns, diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go index 52c8ab300f..005112b4df 100644 --- a/pkg/config/v1alpha1/types.go +++ b/pkg/config/v1alpha1/types.go @@ -135,6 +135,10 @@ type ControllerHealth struct { // LivenessEndpointName, defaults to "healthz" // +optional LivenessEndpointName string `json:"livenessEndpointName,omitempty"` + + // StartupEndpointName, defaults to "startz" + // +optional + StartupEndpointName string `json:"startupEndpointName,omitempty"` } // ControllerWebhook defines the webhook server for the controller. diff --git a/pkg/healthz/doc.go b/pkg/healthz/doc.go index 9827eeafed..e1fb1e4686 100644 --- a/pkg/healthz/doc.go +++ b/pkg/healthz/doc.go @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package healthz contains helpers from supporting liveness and readiness endpoints. -// (often referred to as healthz and readyz, respectively). +// Package healthz contains helpers from supporting liveness, readiness and startup endpoints. +// (often referred to as healthz, readyz and startz, respectively). // // This package draws heavily from the apiserver's healthz package // ( https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz ) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index a16f354a1b..1de1a4cf2b 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -56,6 +56,7 @@ const ( defaultReadinessEndpoint = "/readyz" defaultLivenessEndpoint = "/healthz" + defaultStartupEndpoint = "/startz" ) var _ Runnable = &controllerManager{} @@ -94,12 +95,18 @@ type controllerManager struct { // Liveness probe endpoint name livenessEndpointName string + // Startup probe endpoint name + startupEndpointName string + // Readyz probe handler readyzHandler *healthz.Handler // Healthz probe handler healthzHandler *healthz.Handler + // Startz probe handler + startzHandler *healthz.Handler + // pprofListener is used to serve pprof pprofListener net.Listener @@ -213,6 +220,23 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) return nil } +// AddStartzCheck allows you to add Startz checker. +func (cm *controllerManager) AddStartzCheck(name string, check healthz.Checker) error { + cm.Lock() + defer cm.Unlock() + + if cm.started { + return fmt.Errorf("unable to add new checker because healthz endpoint has already been created") + } + + if cm.startzHandler == nil { + cm.startzHandler = &healthz.Handler{Checks: map[string]healthz.Checker{}} + } + + cm.startzHandler.Checks[name] = check + return nil +} + func (cm *controllerManager) GetHTTPClient() *http.Client { return cm.cluster.GetHTTPClient() } @@ -283,6 +307,11 @@ func (cm *controllerManager) addHealthProbeServer() error { // Append '/' suffix to handle subpaths mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) } + if cm.startzHandler != nil { + mux.Handle(cm.startupEndpointName, http.StripPrefix(cm.startupEndpointName, cm.startzHandler)) + // Append '/' suffix to handle subpaths + mux.Handle(cm.startupEndpointName+"/", http.StripPrefix(cm.startupEndpointName, cm.startzHandler)) + } return cm.add(&server{ Kind: "health probe", diff --git a/pkg/manager/internal/integration/manager_test.go b/pkg/manager/internal/integration/manager_test.go index 624aa69339..a4fe432a85 100644 --- a/pkg/manager/internal/integration/manager_test.go +++ b/pkg/manager/internal/integration/manager_test.go @@ -158,6 +158,7 @@ var _ = Describe("manger.Manager Start", func() { // Configure health probes. Expect(mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) Expect(mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) + Expect(mgr.AddStartzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) // Set up Driver reconciler (using v2). driverReconciler := &DriverReconciler{ diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 25c3c7375b..a8d544e91c 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -73,6 +73,9 @@ type Manager interface { // AddReadyzCheck allows you to add Readyz checker AddReadyzCheck(name string, check healthz.Checker) error + // AddStartzCheck allows you to add Startz checker + AddStartzCheck(name string, check healthz.Checker) error + // Start starts all registered Controllers and blocks until the context is cancelled. // Returns an error if there is an error starting any controller. // @@ -228,6 +231,9 @@ type Options struct { // Liveness probe endpoint name, defaults to "healthz" LivenessEndpointName string + // Startup probe endpoint name, defaults to "healthz" + StartupEndpointName string + // PprofBindAddress is the TCP address that the controller should bind to // for serving pprof. // It can be set to "" or "0" to disable the pprof serving. @@ -430,6 +436,7 @@ func New(config *rest.Config, options Options) (Manager, error) { healthProbeListener: healthProbeListener, readinessEndpointName: options.ReadinessEndpointName, livenessEndpointName: options.LivenessEndpointName, + startupEndpointName: options.StartupEndpointName, pprofListener: pprofListener, gracefulShutdownTimeout: *options.GracefulShutdownTimeout, internalProceduresStop: make(chan struct{}), @@ -480,6 +487,10 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, o.LivenessEndpointName = newObj.Health.LivenessEndpointName } + if o.StartupEndpointName == "" && newObj.Health.StartupEndpointName != "" { + o.StartupEndpointName = newObj.Health.StartupEndpointName + } + if o.WebhookServer == nil { port := 0 if newObj.Webhook.Port != nil { @@ -640,6 +651,10 @@ func setOptionsDefaults(options Options) Options { options.LivenessEndpointName = defaultLivenessEndpoint } + if options.StartupEndpointName == "" { + options.StartupEndpointName = defaultStartupEndpoint + } + if options.newHealthProbeListener == nil { options.newHealthProbeListener = defaultHealthProbeListener } diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 90596e9ace..4fbaf6ea50 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -145,6 +145,7 @@ var _ = Describe("manger.Manager", func() { HealthProbeBindAddress: "6060", ReadinessEndpointName: "/readyz", LivenessEndpointName: "/livez", + StartupEndpointName: "/startz", }, Webhook: v1alpha1.ControllerWebhook{ Port: &port, @@ -170,6 +171,7 @@ var _ = Describe("manger.Manager", func() { Expect(m.HealthProbeBindAddress).To(Equal("6060")) Expect(m.ReadinessEndpointName).To(Equal("/readyz")) Expect(m.LivenessEndpointName).To(Equal("/livez")) + Expect(m.StartupEndpointName).To(Equal("/startz")) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Port).To(Equal(port)) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Host).To(Equal("localhost")) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.CertDir).To(Equal("/certs")) @@ -201,6 +203,7 @@ var _ = Describe("manger.Manager", func() { HealthProbeBindAddress: "6060", ReadinessEndpointName: "/readyz", LivenessEndpointName: "/livez", + StartupEndpointName: "/startz", }, Webhook: v1alpha1.ControllerWebhook{ Port: &port, @@ -229,6 +232,7 @@ var _ = Describe("manger.Manager", func() { HealthProbeBindAddress: "5000", ReadinessEndpointName: "/readiness", LivenessEndpointName: "/liveness", + StartupEndpointName: "/startup", WebhookServer: webhook.NewServer(webhook.Options{ Port: 8080, Host: "example.com", @@ -251,6 +255,7 @@ var _ = Describe("manger.Manager", func() { Expect(m.HealthProbeBindAddress).To(Equal("5000")) Expect(m.ReadinessEndpointName).To(Equal("/readiness")) Expect(m.LivenessEndpointName).To(Equal("/liveness")) + Expect(m.StartupEndpointName).To(Equal("/startup")) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Port).To(Equal(8080)) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Host).To(Equal("example.com")) Expect(m.WebhookServer.(*webhook.DefaultServer).Options.CertDir).To(Equal("/pki")) @@ -1399,6 +1404,11 @@ var _ = Describe("manger.Manager", func() { }) Context("should start serving health probes", func() { + + const ( + namedCheck = "check" + ) + var listener net.Listener var opts Options @@ -1452,7 +1462,6 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) res := fmt.Errorf("not ready yet") - namedCheck := "check" err = m.AddReadyzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) @@ -1507,7 +1516,6 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) res := fmt.Errorf("not alive") - namedCheck := "check" err = m.AddHealthzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) @@ -1547,7 +1555,7 @@ var _ = Describe("manger.Manager", func() { defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) - // Check readiness path for individual check + // Check liveness path for individual check livenessEndpoint = fmt.Sprint("http://", listener.Addr().String(), path.Join(defaultLivenessEndpoint, namedCheck)) res = nil resp, err = http.Get(livenessEndpoint) @@ -1555,6 +1563,60 @@ var _ = Describe("manger.Manager", func() { defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) + + It("should serve startup endpoint", func() { + opts.HealthProbeBindAddress = ":0" + m, err := New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + res := fmt.Errorf("not alive") + err = m.AddStartzCheck(namedCheck, func(_ *http.Request) error { return res }) + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + + startupEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultStartupEndpoint) + + // Controller is not ready + resp, err := http.Get(startupEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) + + // Controller is ready + res = nil + resp, err = http.Get(startupEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + // Check startup path without trailing slash without redirect + startupEndpoint = fmt.Sprint("http://", listener.Addr().String(), defaultStartupEndpoint) + res = nil + httpClient := http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse // Do not follow redirect + }, + } + resp, err = httpClient.Get(startupEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + // Check startup path for individual check + startupEndpoint = fmt.Sprint("http://", listener.Addr().String(), path.Join(defaultStartupEndpoint, namedCheck)) + res = nil + resp, err = http.Get(startupEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) }) Context("should start serving pprof", func() {