diff --git a/cmd/koord-manager/main.go b/cmd/koord-manager/main.go index 96be54c16..e4212b179 100644 --- a/cmd/koord-manager/main.go +++ b/cmd/koord-manager/main.go @@ -24,10 +24,12 @@ import ( "os" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" @@ -36,6 +38,7 @@ import ( "github.com/koordinator-sh/koordinator/cmd/koord-manager/options" extclient "github.com/koordinator-sh/koordinator/pkg/client" "github.com/koordinator-sh/koordinator/pkg/features" + "github.com/koordinator-sh/koordinator/pkg/slo-controller/metrics" utilclient "github.com/koordinator-sh/koordinator/pkg/util/client" utilfeature "github.com/koordinator-sh/koordinator/pkg/util/feature" "github.com/koordinator-sh/koordinator/pkg/util/fieldindex" @@ -170,6 +173,11 @@ func main() { klog.V(4).Infof("webhook framework feature gate not enabled") } + if err := installHTTPHandler(mgr); err != nil { + setupLog.Error(err, "unable to install http handler") + os.Exit(1) + } + setupLog.Info("starting manager") extensions.StartExtensions(ctx, mgr) if err := mgr.Start(ctx); err != nil { @@ -186,3 +194,14 @@ func setRestConfig(c *rest.Config) { c.Burst = *restConfigBurst } } + +func installHTTPHandler(mgr ctrl.Manager) error { + // consider mv legacyregistry.metrics into controller-runtime.metrics + if err := mgr.AddMetricsExtraHandler(metrics.InternalHTTPPath, promhttp.HandlerFor(metrics.InternalRegistry, promhttp.HandlerOpts{})); err != nil { + return err + } + if err := mgr.AddMetricsExtraHandler(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{})); err != nil { + return err + } + return nil +} diff --git a/cmd/koordlet/main.go b/cmd/koordlet/main.go index b2df9fcd6..f04a483bc 100644 --- a/cmd/koordlet/main.go +++ b/cmd/koordlet/main.go @@ -33,6 +33,7 @@ import ( agent "github.com/koordinator-sh/koordinator/pkg/koordlet" "github.com/koordinator-sh/koordinator/pkg/koordlet/audit" "github.com/koordinator-sh/koordinator/pkg/koordlet/config" + "github.com/koordinator-sh/koordinator/pkg/koordlet/metrics" ) func main() { @@ -77,18 +78,23 @@ func main() { } // Expose the Prometheus http endpoint - go func() { - klog.Infof("Starting prometheus server on %v", *options.ServerAddr) - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) { - mux.HandleFunc("/events", audit.HttpHandler()) - } - // http.HandleFunc("/healthz", d.HealthzHandler()) - klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux)) - }() + go installHTTPHandler() // Start the Cmd klog.Info("Starting the koordlet daemon") d.Run(stopCtx.Done()) } + +func installHTTPHandler() { + klog.Infof("Starting prometheus server on %v", *options.ServerAddr) + mux := http.NewServeMux() + mux.Handle(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{})) + mux.Handle(metrics.InternalHTTPPath, promhttp.HandlerFor(metrics.InternalRegistry, promhttp.HandlerOpts{})) + // compatibility purpose for old http url + mux.Handle(metrics.OldExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{})) + if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) { + mux.HandleFunc("/events", audit.HttpHandler()) + } + // http.HandleFunc("/healthz", d.HealthzHandler()) + klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux)) +} diff --git a/pkg/koordlet/metrics/external_metrics.go b/pkg/koordlet/metrics/external_metrics.go new file mode 100644 index 000000000..4761b7bd9 --- /dev/null +++ b/pkg/koordlet/metrics/external_metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const ( + ExternalHTTPPath = "/external-metrics" + OldExternalHTTPPath = "/metrics" +) + +var ( + // ExternalRegistry register metrics for users such as PMU or extended resources settings + ExternalRegistry = prometheus.NewRegistry() +) + +func ExternalMustRegister(metrics ...prometheus.Collector) { + ExternalRegistry.MustRegister(metrics...) +} + +func init() { + ExternalMustRegister(ResourceSummaryCollectors...) + ExternalMustRegister(CPICollectors...) + ExternalMustRegister(PSICollectors...) +} diff --git a/pkg/koordlet/metrics/internal_metrics.go b/pkg/koordlet/metrics/internal_metrics.go new file mode 100644 index 000000000..f9b932752 --- /dev/null +++ b/pkg/koordlet/metrics/internal_metrics.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics/legacyregistry" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics +) + +const ( + InternalHTTPPath = "/internal-metrics" +) + +var ( + // InternalRegistry only register metrics of koordlet itself for performance and functional monitor + // TODO consider using k8s.io/component-base/metrics to replace github.com/prometheus/client_golang/prometheus + InternalRegistry = legacyregistry.DefaultGatherer +) + +func internalMustRegister(metrics ...prometheus.Collector) { + legacyregistry.RawMustRegister(metrics...) +} + +func init() { + internalMustRegister(CommonCollectors...) + internalMustRegister(CPUSuppressCollector...) + internalMustRegister(CPUBurstCollector...) + internalMustRegister(PredictionCollectors...) + internalMustRegister(CoreSchedCollector...) +} diff --git a/pkg/koordlet/metrics/metrics.go b/pkg/koordlet/metrics/metrics.go index aaff8c645..de0f4b9e8 100644 --- a/pkg/koordlet/metrics/metrics.go +++ b/pkg/koordlet/metrics/metrics.go @@ -24,17 +24,6 @@ import ( "k8s.io/klog/v2" ) -func init() { - prometheus.MustRegister(CommonCollectors...) - prometheus.MustRegister(ResourceSummaryCollectors...) - prometheus.MustRegister(CPICollectors...) - prometheus.MustRegister(PSICollectors...) - prometheus.MustRegister(CPUSuppressCollector...) - prometheus.MustRegister(CPUBurstCollector...) - prometheus.MustRegister(PredictionCollectors...) - prometheus.MustRegister(CoreSchedCollector...) -} - const ( KoordletSubsystem = "koordlet" diff --git a/pkg/slo-controller/metrics/common.go b/pkg/slo-controller/metrics/common.go index 360ab6bd6..d21b5091b 100644 --- a/pkg/slo-controller/metrics/common.go +++ b/pkg/slo-controller/metrics/common.go @@ -19,7 +19,7 @@ package metrics import "github.com/prometheus/client_golang/prometheus" func init() { - MustRegister(CommonCollectors...) + InternalMustRegister(CommonCollectors...) } var ( diff --git a/pkg/slo-controller/metrics/metrics.go b/pkg/slo-controller/metrics/metrics.go index 382787830..0c3d4b2b5 100644 --- a/pkg/slo-controller/metrics/metrics.go +++ b/pkg/slo-controller/metrics/metrics.go @@ -19,7 +19,8 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/metrics" + "k8s.io/component-base/metrics/legacyregistry" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics ) const ( @@ -41,11 +42,25 @@ const ( UnitInteger = "integer" ) -// DefaultRegistry uses the controller runtime registry by default. -var DefaultRegistry = metrics.Registry +const ( + ExternalHTTPPath = "/external-metrics" + InternalHTTPPath = "/internal-metrics" +) + +var ( + // ExternalRegistry register metrics for users + ExternalRegistry = prometheus.NewRegistry() + + // InternalRegistry only register metrics of koord-manager itself for performance and functional monitor + InternalRegistry = legacyregistry.DefaultGatherer +) + +func ExternalMustRegister(cs ...prometheus.Collector) { + ExternalRegistry.MustRegister(cs...) +} -func MustRegister(cs ...prometheus.Collector) { - DefaultRegistry.MustRegister(cs...) +func InternalMustRegister(cs ...prometheus.Collector) { + legacyregistry.RawMustRegister(cs...) } func genNodeLabels(node *corev1.Node) prometheus.Labels { diff --git a/pkg/slo-controller/metrics/metrics_test.go b/pkg/slo-controller/metrics/metrics_test.go index 49e85e3f6..f320dc8e2 100644 --- a/pkg/slo-controller/metrics/metrics_test.go +++ b/pkg/slo-controller/metrics/metrics_test.go @@ -35,7 +35,16 @@ func TestMustRegister(t *testing.T) { Help: "test counter", }, []string{StatusKey, ReasonKey}) assert.NotPanics(t, func() { - MustRegister(testMetricVec) + InternalMustRegister(testMetricVec) + }) + + testExternalMetricVec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: "test", + Name: "test_external_counter", + Help: "test counter", + }, []string{StatusKey, ReasonKey}) + assert.NotPanics(t, func() { + ExternalMustRegister(testExternalMetricVec) }) } diff --git a/pkg/slo-controller/metrics/node_resource.go b/pkg/slo-controller/metrics/node_resource.go index 908a0fd5c..e49e3da3f 100644 --- a/pkg/slo-controller/metrics/node_resource.go +++ b/pkg/slo-controller/metrics/node_resource.go @@ -24,7 +24,7 @@ import ( ) func init() { - MustRegister(NodeResourceCollectors...) + InternalMustRegister(NodeResourceCollectors...) } var (