Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

⚠ Remove metrics with high cardinality #2298

Merged
merged 1 commit into from
May 4, 2023
+1 −88
Merged

⚠ Remove metrics with high cardinality #2298

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 1 addition & 88 deletions pkg/metrics/client_go_adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ package metrics

import (
"context"
"net/url"
"time"

"github.com/prometheus/client_golang/prometheus"
clientmetrics "k8s.io/client-go/tools/metrics"
Expand All @@ -29,70 +27,9 @@ import (
// that client-go registers metrics. We copy the names and formats
// from Kubernetes so that we match the core controllers.

// Metrics subsystem and all of the keys used by the rest client.
const (
RestClientSubsystem = "rest_client"
LatencyKey = "request_latency_seconds"
ResultKey = "requests_total"
)

var (
// client metrics.

// RequestLatency reports the request latency in seconds per verb/URL.
// Deprecated: This metric is deprecated for removal in a future release: using the URL as a
// dimension results in cardinality explosion for some consumers. It was deprecated upstream
// in k8s v1.14 and hidden in v1.17 via https://github.com/kubernetes/kubernetes/pull/83836.
// It is not registered by default. To register:
// import (
// clientmetrics "k8s.io/client-go/tools/metrics"
// clmetrics "sigs.k8s.io/controller-runtime/metrics"
// )
//
// func init() {
// clmetrics.Registry.MustRegister(clmetrics.RequestLatency)
// clientmetrics.Register(clientmetrics.RegisterOpts{
// RequestLatency: clmetrics.LatencyAdapter
// })
// }
RequestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: RestClientSubsystem,
Name: LatencyKey,
Help: "Request latency in seconds. Broken down by verb and URL.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 10),
}, []string{"verb", "url"})

// requestLatency is a Prometheus Histogram metric type partitioned by
// "verb", and "host" labels. It is used for the rest client latency metrics.
requestLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_request_duration_seconds",
Help: "Request latency in seconds. Broken down by verb, and host.",
Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0},
},
[]string{"verb", "host"},
)

requestSize = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_request_size_bytes",
Help: "Request size in bytes. Broken down by verb and host.",
// 64 bytes to 16MB
Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
},
[]string{"verb", "host"},
)

responseSize = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_response_size_bytes",
Help: "Response size in bytes. Broken down by verb and host.",
// 64 bytes to 16MB
Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
},
[]string{"verb", "host"},
)

requestResult = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "rest_client_requests_total",
Expand All @@ -109,17 +46,11 @@ func init() {
// registerClientMetrics sets up the client latency metrics from client-go.
func registerClientMetrics() {
// register the metrics with our registry
Registry.MustRegister(requestLatency)
Registry.MustRegister(requestSize)
Registry.MustRegister(responseSize)
Registry.MustRegister(requestResult)

// register the metrics with client-go
clientmetrics.Register(clientmetrics.RegisterOpts{
RequestLatency: &LatencyAdapter{metric: requestLatency},
RequestSize: &sizeAdapter{metric: requestSize},
ResponseSize: &sizeAdapter{metric: responseSize},
RequestResult: &resultAdapter{metric: requestResult},
RequestResult: &resultAdapter{metric: requestResult},
})
}

Expand All @@ -131,24 +62,6 @@ func registerClientMetrics() {
// copied (more-or-less directly) from k8s.io/kubernetes setup code
// (which isn't anywhere in an easily-importable place).

// LatencyAdapter implements LatencyMetric.
type LatencyAdapter struct {
metric *prometheus.HistogramVec
}

// Observe increments the request latency metric for the given verb/URL.
func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
}

type sizeAdapter struct {
metric *prometheus.HistogramVec
}

func (s *sizeAdapter) Observe(ctx context.Context, verb string, host string, size float64) {
s.metric.WithLabelValues(verb, host).Observe(size)
}

type resultAdapter struct {
metric *prometheus.CounterVec
}
Expand Down