metrics: seperater metrics as internal and external for slo-controller and koordlet

cmd/koord-manager/main.go

@@ -24,21 +24,26 @@ import (
 	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/pflag"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/koordinator-sh/koordinator/cmd/koord-manager/extensions"
 	"github.com/koordinator-sh/koordinator/cmd/koord-manager/options"
 	extclient "github.com/koordinator-sh/koordinator/pkg/client"
 	"github.com/koordinator-sh/koordinator/pkg/features"
+	"github.com/koordinator-sh/koordinator/pkg/slo-controller/metrics"
 	utilclient "github.com/koordinator-sh/koordinator/pkg/util/client"
 	utilfeature "github.com/koordinator-sh/koordinator/pkg/util/feature"
 	"github.com/koordinator-sh/koordinator/pkg/util/fieldindex"
+	metricsutil "github.com/koordinator-sh/koordinator/pkg/util/metrics"
 	_ "github.com/koordinator-sh/koordinator/pkg/util/metrics/leadership"
 	"github.com/koordinator-sh/koordinator/pkg/util/sloconfig"
 	"github.com/koordinator-sh/koordinator/pkg/webhook"
@@ -170,6 +175,11 @@ func main() {
 		klog.V(4).Infof("webhook framework feature gate not enabled")
 	}
 
+	if err := installHTTPHandler(mgr); err != nil {
+		setupLog.Error(err, "unable to install http handler")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	extensions.StartExtensions(ctx, mgr)
 	if err := mgr.Start(ctx); err != nil {
@@ -186,3 +196,18 @@ func setRestConfig(c *rest.Config) {
 		c.Burst = *restConfigBurst
 	}
 }
+
+func installHTTPHandler(mgr ctrl.Manager) error {
+	if err := mgr.AddMetricsExtraHandler(metrics.InternalHTTPPath, promhttp.HandlerFor(metrics.InternalRegistry, promhttp.HandlerOpts{})); err != nil {
+		return err
+	}
+	if err := mgr.AddMetricsExtraHandler(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{})); err != nil {
+		return err
+	}
+	// merge internal, external and controller-runtime metrics
+	if err := mgr.AddMetricsExtraHandler(metrics.DefaultHTTPPath, promhttp.HandlerFor(
+		metricsutil.MergedGatherFunc(metrics.InternalRegistry, metrics.ExternalRegistry, ctrlmetrics.Registry), promhttp.HandlerOpts{})); err != nil {
+		return err
+	}
+	return nil
+}

cmd/koordlet/main.go

@@ -33,6 +33,8 @@ import (
 	agent "github.com/koordinator-sh/koordinator/pkg/koordlet"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/audit"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/config"
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/metrics"
+	metricsutil "github.com/koordinator-sh/koordinator/pkg/util/metrics"
 )
 
 func main() {
@@ -77,18 +79,24 @@ func main() {
 	}
 
 	// Expose the Prometheus http endpoint
-	go func() {
-		klog.Infof("Starting prometheus server on %v", *options.ServerAddr)
-		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.Handler())
-		if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) {
-			mux.HandleFunc("/events", audit.HttpHandler())
-		}
-		// http.HandleFunc("/healthz", d.HealthzHandler())
-		klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux))
-	}()
+	go installHTTPHandler()
 
 	// Start the Cmd
 	klog.Info("Starting the koordlet daemon")
 	d.Run(stopCtx.Done())
 }
+
+func installHTTPHandler() {
+	klog.Infof("Starting prometheus server on %v", *options.ServerAddr)
+	mux := http.NewServeMux()
+	mux.Handle(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{}))
+	mux.Handle(metrics.InternalHTTPPath, promhttp.HandlerFor(metrics.InternalRegistry, promhttp.HandlerOpts{}))
+	// merge internal and external
+	mux.Handle(metrics.DefaultHTTPPath, promhttp.HandlerFor(
+		metricsutil.MergedGatherFunc(metrics.InternalRegistry, metrics.ExternalRegistry), promhttp.HandlerOpts{}))
+	if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) {
+		mux.HandleFunc("/events", audit.HttpHandler())
+	}
+	// http.HandleFunc("/healthz", d.HealthzHandler())
+	klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux))
+}

pkg/koordlet/metrics/external_metrics.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	ExternalHTTPPath = "/external-metrics"
+)
+
+var (
+	// ExternalRegistry	register metrics for users such as PMU or extended resources settings
+	ExternalRegistry = prometheus.NewRegistry()
+)
+
+func ExternalMustRegister(metrics ...prometheus.Collector) {
+	ExternalRegistry.MustRegister(metrics...)
+}
+
+func init() {
+	ExternalMustRegister(ResourceSummaryCollectors...)
+	ExternalMustRegister(CPICollectors...)
+	ExternalMustRegister(PSICollectors...)
+}

pkg/koordlet/metrics/internal_metrics.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	InternalHTTPPath = "/internal-metrics"
+)
+
+var (
+	// InternalRegistry only register metrics of koordlet itself for performance and functional monitor
+	// TODO consider using k8s.io/component-base/metrics to replace github.com/prometheus/client_golang/prometheus
+	InternalRegistry = legacyregistry.DefaultGatherer
+)
+
+func internalMustRegister(metrics ...prometheus.Collector) {
+	legacyregistry.RawMustRegister(metrics...)
+}
+
+func init() {
+	internalMustRegister(CommonCollectors...)
+	internalMustRegister(CPUSuppressCollector...)
+	internalMustRegister(CPUBurstCollector...)
+	internalMustRegister(PredictionCollectors...)
+	internalMustRegister(CoreSchedCollector...)
+}

pkg/koordlet/metrics/metrics.go

@@ -24,17 +24,6 @@ import (
 	"k8s.io/klog/v2"
 )
 
-func init() {
-	prometheus.MustRegister(CommonCollectors...)
-	prometheus.MustRegister(ResourceSummaryCollectors...)
-	prometheus.MustRegister(CPICollectors...)
-	prometheus.MustRegister(PSICollectors...)
-	prometheus.MustRegister(CPUSuppressCollector...)
-	prometheus.MustRegister(CPUBurstCollector...)
-	prometheus.MustRegister(PredictionCollectors...)
-	prometheus.MustRegister(CoreSchedCollector...)
-}
-
 const (
 	KoordletSubsystem = "koordlet"
 
@@ -72,6 +61,10 @@ var (
 	nodeLock sync.RWMutex
 )
 
+const (
+	DefaultHTTPPath = "/metrics"
+)
+
 // Register registers the metrics with the node object
 func Register(node *corev1.Node) {
 	nodeLock.Lock()

pkg/slo-controller/metrics/common.go

@@ -19,7 +19,7 @@ package metrics
 import "github.com/prometheus/client_golang/prometheus"
 
 func init() {
-	MustRegister(CommonCollectors...)
+	InternalMustRegister(CommonCollectors...)
 }
 
 var (

pkg/slo-controller/metrics/metrics.go

@@ -19,7 +19,8 @@ package metrics
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics
 )
 
 const (
@@ -41,11 +42,27 @@ const (
 	UnitInteger = "integer"
 )
 
-// DefaultRegistry uses the controller runtime registry by default.
-var DefaultRegistry = metrics.Registry
+const (
+	// DefaultHTTPPath use /all-metrics since /metrics is occupied by controller manager default registry
+	DefaultHTTPPath  = "/all-metrics"
+	ExternalHTTPPath = "/external-metrics"
+	InternalHTTPPath = "/internal-metrics"
+)
+
+var (
+	// ExternalRegistry	register metrics for users
+	ExternalRegistry = prometheus.NewRegistry()
+
+	// InternalRegistry only register metrics of koord-manager itself for performance and functional monitor
+	InternalRegistry = legacyregistry.DefaultGatherer
+)
+
+func ExternalMustRegister(cs ...prometheus.Collector) {
+	ExternalRegistry.MustRegister(cs...)
+}
 
-func MustRegister(cs ...prometheus.Collector) {
-	DefaultRegistry.MustRegister(cs...)
+func InternalMustRegister(cs ...prometheus.Collector) {
+	legacyregistry.RawMustRegister(cs...)
 }
 
 func genNodeLabels(node *corev1.Node) prometheus.Labels {

pkg/slo-controller/metrics/metrics_test.go

@@ -35,7 +35,16 @@ func TestMustRegister(t *testing.T) {
 		Help:      "test counter",
 	}, []string{StatusKey, ReasonKey})
 	assert.NotPanics(t, func() {
-		MustRegister(testMetricVec)
+		InternalMustRegister(testMetricVec)
+	})
+
+	testExternalMetricVec := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: "test",
+		Name:      "test_external_counter",
+		Help:      "test counter",
+	}, []string{StatusKey, ReasonKey})
+	assert.NotPanics(t, func() {
+		ExternalMustRegister(testExternalMetricVec)
 	})
 }
 

pkg/slo-controller/metrics/node_resource.go

@@ -24,7 +24,7 @@ import (
 )
 
 func init() {
-	MustRegister(NodeResourceCollectors...)
+	InternalMustRegister(NodeResourceCollectors...)
 }
 
 var (

pkg/util/metrics/merged_gather.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+var _ prometheus.Gatherer = &mergedGather{}
+
+type mergedGather struct {
+	gathers []prometheus.Gatherer
+}
+
+// MergedGatherFunc returns a Gatherer that merges the results of multiple Gatherers
+func MergedGatherFunc(g ...prometheus.Gatherer) prometheus.Gatherer {
+	return &mergedGather{gathers: g}
+}
+
+func (m *mergedGather) Gather() ([]*dto.MetricFamily, error) {
+	result := make([]*dto.MetricFamily, 0)
+	for _, g := range m.gathers {
+		if metrics, err := g.Gather(); err != nil {
+			return result, err
+		} else {
+			result = append(result, metrics...)
+		}
+	}
+	return result, nil
+}