From 93af2e139df6a72027ed721a73f894ebec9f4e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Tue, 2 Jul 2024 09:53:11 +0200 Subject: [PATCH] feat: allow adding new prometheus.Gatherers on top of default registry In case of libraries using prebuilt registries (such as controller runtime itself), it should be possible to add the gatherer to the default metrics server so that no second handler or path has to be created for them. This allows to add metrics that are precollected/registered in something outside of the own controller runtime registry --- pkg/manager/internal.go | 17 ++++++++++++++ pkg/manager/manager.go | 5 ++++ pkg/manager/manager_test.go | 44 ++++++++++++++++++++++++++++++++++++ pkg/metrics/server/server.go | 17 +++++++++++++- 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index 2ce02b105c..baf2ca47ec 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -28,6 +28,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -197,6 +198,22 @@ func (cm *controllerManager) AddMetricsServerExtraHandler(path string, handler h return nil } +// AddMetricsServerExtraGatherer adds an extra prometheus.Gatherer to the metrics server. +func (cm *controllerManager) AddMetricsServerExtraGatherer(gatherer prometheus.Gatherer) error { + cm.Lock() + defer cm.Unlock() + if cm.started { + return fmt.Errorf("unable to add new gatherer because metrics endpoint has already been created") + } + if cm.metricsServer == nil { + cm.GetLogger().Info("warn: metrics server is currently disabled, registering extra gatherer will be ignored") + return nil + } + cm.metricsServer.AddExtraGatherer(gatherer) + cm.logger.V(2).Info("Registering metrics http server extra gatherer") + return nil +} + // AddHealthzCheck allows you to add Healthz checker. func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error { cm.Lock() diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 3166f4818f..4af9f12c56 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -25,6 +25,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -73,6 +74,10 @@ type Manager interface { // a new http server/listener should be added as Runnable to the manager via Add method. AddMetricsServerExtraHandler(path string, handler http.Handler) error + // AddMetricsServerExtraGatherer adds an extra prometheus.Gatherer to the metrics server. + // This can be used to add custom metrics to the metrics server. + AddMetricsServerExtraGatherer(gatherer prometheus.Gatherer) error + // AddHealthzCheck allows you to add Healthz checker AddHealthzCheck(name string, check healthz.Checker) error diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 1683013b3f..47276349a6 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -1303,6 +1303,50 @@ var _ = Describe("manger.Manager", func() { Expect(ok).To(BeTrue()) }) + It("should serve metrics in an extra registry", func() { + extraRegistry := prometheus.NewRegistry() + one := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "test_one", + Help: "test metric for testing", + }) + one.Inc() + err := extraRegistry.Register(one) + Expect(err).NotTo(HaveOccurred()) + + m, err := New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + Expect(m.AddMetricsServerExtraGatherer(extraRegistry)).To(Succeed()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) + + metricsEndpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) + resp, err := http.Get(metricsEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(200)) + + data, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n", + `# HELP test_one test metric for testing`, + `# TYPE test_one counter`, + `test_one 1`, + )) + + // Unregister will return false if the metric was never registered + ok := extraRegistry.Unregister(one) + Expect(ok).To(BeTrue()) + }) + It("should serve extra endpoints", func() { opts.Metrics.ExtraHandlers = map[string]http.Handler{ "/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { diff --git a/pkg/metrics/server/server.go b/pkg/metrics/server/server.go index 5eb0c62a72..d27acfde8d 100644 --- a/pkg/metrics/server/server.go +++ b/pkg/metrics/server/server.go @@ -28,6 +28,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/client-go/rest" certutil "k8s.io/client-go/util/cert" @@ -49,6 +50,9 @@ type Server interface { // AddExtraHandler adds extra handler served on path to the http server that serves metrics. AddExtraHandler(path string, handler http.Handler) error + // AddExtraGatherer adds a gatherer to the metrics server that is used on top of the default via metrics.Registry. + AddExtraGatherer(gatherer prometheus.Gatherer) + // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates // the metrics server doesn't need leader election. NeedLeaderElection() bool @@ -76,6 +80,11 @@ type Options struct { // server/listener should be added as Runnable to the manager via the Add method. ExtraHandlers map[string]http.Handler + // ExtraGatherers contains a list of gatherers that will be added to the metrics server. + // This might be useful to add custom metrics to the metrics server that are not registerable + // via the metrics.Registry directly, such as prebuilt registries from libraries. + ExtraGatherers []prometheus.Gatherer + // FilterProvider provides a filter which is a func that is added around // the metrics and the extra handlers on the metrics server. // This can be e.g. used to enforce authentication and authorization on the handlers @@ -202,6 +211,10 @@ func (s *defaultServer) AddExtraHandler(path string, handler http.Handler) error return nil } +func (s *defaultServer) AddExtraGatherer(gatherer prometheus.Gatherer) { + s.options.ExtraGatherers = append(s.options.ExtraGatherers, gatherer) +} + // Start runs the server. // It will install the metrics related resources depend on the server configuration. func (s *defaultServer) Start(ctx context.Context) error { @@ -218,7 +231,9 @@ func (s *defaultServer) Start(ctx context.Context) error { mux := http.NewServeMux() - handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ + gatherer := prometheus.Gatherers(append([]prometheus.Gatherer{metrics.Registry}, s.options.ExtraGatherers...)) + + handler := promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{ ErrorHandling: promhttp.HTTPErrorOnError, }) if s.metricsFilter != nil {