koordlet/koord-manager: seperater metrics as internal and external

Signed-off-by: 佑祎 <zzw261520@alibaba-inc.com>

cmd/koord-manager/main.go

@@ -24,10 +24,12 @@ import (
 	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/pflag"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -36,6 +38,7 @@ import (
 	"github.com/koordinator-sh/koordinator/cmd/koord-manager/options"
 	extclient "github.com/koordinator-sh/koordinator/pkg/client"
 	"github.com/koordinator-sh/koordinator/pkg/features"
+	"github.com/koordinator-sh/koordinator/pkg/slo-controller/metrics"
 	utilclient "github.com/koordinator-sh/koordinator/pkg/util/client"
 	utilfeature "github.com/koordinator-sh/koordinator/pkg/util/feature"
 	"github.com/koordinator-sh/koordinator/pkg/util/fieldindex"
@@ -170,6 +173,11 @@ func main() {
 		klog.V(4).Infof("webhook framework feature gate not enabled")
 	}
 
+	if err := installHTTPHandler(mgr); err != nil {
+		setupLog.Error(err, "unable to install http handler")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	extensions.StartExtensions(ctx, mgr)
 	if err := mgr.Start(ctx); err != nil {
@@ -186,3 +194,14 @@ func setRestConfig(c *rest.Config) {
 		c.Burst = *restConfigBurst
 	}
 }
+
+func installHTTPHandler(mgr ctrl.Manager) error {
+	// consider mv legacyregistry.metrics into controller-runtime.metrics
+	if err := mgr.AddMetricsExtraHandler(metrics.LegacyHTTPPath, promhttp.HandlerFor(legacyregistry.DefaultGatherer, promhttp.HandlerOpts{})); err != nil {
+		return err
+	}
+	if err := mgr.AddMetricsExtraHandler(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{})); err != nil {
+		return err
+	}
+	return nil
+}

cmd/koordlet/main.go

@@ -33,6 +33,7 @@ import (
 	agent "github.com/koordinator-sh/koordinator/pkg/koordlet"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/audit"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/config"
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/metrics"
 )
 
 func main() {
@@ -77,18 +78,21 @@ func main() {
 	}
 
 	// Expose the Prometheus http endpoint
-	go func() {
-		klog.Infof("Starting prometheus server on %v", *options.ServerAddr)
-		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.Handler())
-		if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) {
-			mux.HandleFunc("/events", audit.HttpHandler())
-		}
-		// http.HandleFunc("/healthz", d.HealthzHandler())
-		klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux))
-	}()
+	go installHTTPHandler()
 
 	// Start the Cmd
 	klog.Info("Starting the koordlet daemon")
 	d.Run(stopCtx.Done())
 }
+
+func installHTTPHandler() {
+	klog.Infof("Starting prometheus server on %v", *options.ServerAddr)
+	mux := http.NewServeMux()
+	mux.Handle(metrics.ExternalHTTPPath, promhttp.HandlerFor(metrics.ExternalRegistry, promhttp.HandlerOpts{}))
+	mux.Handle(metrics.InternalHTTPPath, promhttp.HandlerFor(metrics.InternalRegistry, promhttp.HandlerOpts{}))
+	if features.DefaultKoordletFeatureGate.Enabled(features.AuditEventsHTTPHandler) {
+		mux.HandleFunc("/events", audit.HttpHandler())
+	}
+	// http.HandleFunc("/healthz", d.HealthzHandler())
+	klog.Fatalf("Prometheus monitoring failed: %v", http.ListenAndServe(*options.ServerAddr, mux))
+}

pkg/koordlet/metrics/external_metrics.go

@@ -0,0 +1,38 @@
+/*
+ Copyright 2023 The Koordinator Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	ExternalHTTPPath = "/external-metrics"
+)
+
+var (
+	// ExternalRegistry	register metrics for users such as PMU or extended resources settings
+	ExternalRegistry = prometheus.NewRegistry()
+)
+
+func ExternalMustRegister(metrics ...prometheus.Collector) {
+	ExternalRegistry.MustRegister(metrics...)
+}
+
+func init() {
+	ExternalMustRegister(ResourceSummaryCollectors...)
+	ExternalMustRegister(CPICollectors...)
+	ExternalMustRegister(PSICollectors...)
+}

pkg/koordlet/metrics/internal_metrics.go

@@ -0,0 +1,39 @@
+/*
+ Copyright 2023 The Koordinator Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	InternalHTTPPath = "/internal-metrics"
+)
+
+var (
+	// InternalRegistry only register metrics of koordlet itself for performance and functional monitor
+	InternalRegistry = prometheus.NewRegistry()
+)
+
+func internalMustRegister(metrics ...prometheus.Collector) {
+	InternalRegistry.MustRegister(metrics...)
+}
+
+func init() {
+	internalMustRegister(CommonCollectors...)
+	internalMustRegister(CPUSuppressCollector...)
+	internalMustRegister(CPUBurstCollector...)
+	internalMustRegister(PredictionCollectors...)
+}

pkg/koordlet/metrics/metrics.go

@@ -24,16 +24,6 @@ import (
 	"k8s.io/klog/v2"
 )
 
-func init() {
-	prometheus.MustRegister(CommonCollectors...)
-	prometheus.MustRegister(ResourceSummaryCollectors...)
-	prometheus.MustRegister(CPICollectors...)
-	prometheus.MustRegister(PSICollectors...)
-	prometheus.MustRegister(CPUSuppressCollector...)
-	prometheus.MustRegister(CPUBurstCollector...)
-	prometheus.MustRegister(PredictionCollectors...)
-}
-
 const (
 	KoordletSubsystem = "koordlet"
 

pkg/slo-controller/metrics/common.go

@@ -19,7 +19,7 @@ package metrics
 import "github.com/prometheus/client_golang/prometheus"
 
 func init() {
-	MustRegister(CommonCollectors...)
+	InternalMustRegister(CommonCollectors...)
 }
 
 var (

pkg/slo-controller/metrics/metrics.go

@@ -19,6 +19,7 @@ package metrics
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load restclient and workqueue metrics
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
@@ -41,11 +42,26 @@ const (
 	UnitInteger = "integer"
 )
 
-// DefaultRegistry uses the controller runtime registry by default.
-var DefaultRegistry = metrics.Registry
+const (
+	LegacyHTTPPath   = "/legacy-metrics" // for legacyregistry.DefaultGatherer
+	ExternalHTTPPath = "/external-metrics"
+	// InternalHTTPPath = "/metrics"
+)
+
+var (
+	// ExternalRegistry	register metrics for users
+	ExternalRegistry = prometheus.NewRegistry()
+
+	// InternalRegistry only register metrics of koord-manager itself for performance and functional monitor
+	InternalRegistry = metrics.Registry
+)
+
+func ExternalMustRegister(metrics ...prometheus.Collector) {
+	ExternalRegistry.MustRegister(metrics...)
+}
 
-func MustRegister(cs ...prometheus.Collector) {
-	DefaultRegistry.MustRegister(cs...)
+func InternalMustRegister(cs ...prometheus.Collector) {
+	InternalRegistry.MustRegister(cs...)
 }
 
 func genNodeLabels(node *corev1.Node) prometheus.Labels {

pkg/slo-controller/metrics/metrics_test.go

@@ -35,7 +35,16 @@ func TestMustRegister(t *testing.T) {
 		Help:      "test counter",
 	}, []string{StatusKey, ReasonKey})
 	assert.NotPanics(t, func() {
-		MustRegister(testMetricVec)
+		InternalMustRegister(testMetricVec)
+	})
+
+	testExternalMetricVec := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: "test",
+		Name:      "test_external_counter",
+		Help:      "test counter",
+	}, []string{StatusKey, ReasonKey})
+	assert.NotPanics(t, func() {
+		ExternalMustRegister(testExternalMetricVec)
 	})
 }
 

pkg/slo-controller/metrics/node_resource.go

@@ -24,7 +24,7 @@ import (
 )
 
 func init() {
-	MustRegister(NodeResourceCollectors...)
+	InternalMustRegister(NodeResourceCollectors...)
 }
 
 var (