Only keep certain K8s API server metrics by default (#384)

grafana · Feb 4, 2021 · e453249 · e453249
1 parent 2086abf
commit e453249
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 28 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,9 @@ files to the new format.
   that updating a config is guaranteed to always hash to the same Agent,
   reducing the number of metrics gaps. (@rfratto)
 
+- [ENHANCEMENT] Only keep a handful of K8s API server metrics by default to reduce
+  default active series usage. (@hjet)
+
 - [BUGFIX] `agentctl config-check` will now work correctly when the supplied
   config file contains integrations. (@hoenn)
 

diff --git a/production/kubernetes/agent-sigv4.yaml b/production/kubernetes/agent-sigv4.yaml
@@ -207,12 +207,8 @@ data:
                 kubernetes_sd_configs:
                   - role: endpoints
                 metric_relabel_configs:
-                  - action: drop
-                    regex: apiserver_admission_controller_admission_latencies_seconds_.*
-                    source_labels:
-                      - __name__
-                  - action: drop
-                    regex: apiserver_admission_step_admission_latencies_seconds_.*
+                  - action: keep
+                    regex: workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines
                     source_labels:
                       - __name__
                 relabel_configs:

diff --git a/production/kubernetes/agent.yaml b/production/kubernetes/agent.yaml
@@ -205,12 +205,8 @@ data:
                 kubernetes_sd_configs:
                   - role: endpoints
                 metric_relabel_configs:
-                  - action: drop
-                    regex: apiserver_admission_controller_admission_latencies_seconds_.*
-                    source_labels:
-                      - __name__
-                  - action: drop
-                    regex: apiserver_admission_step_admission_latencies_seconds_.*
+                  - action: keep
+                    regex: workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines
                     source_labels:
                       - __name__
                 relabel_configs:

diff --git a/production/tanka/grafana-agent/config.libsonnet b/production/tanka/grafana-agent/config.libsonnet
@@ -121,17 +121,12 @@
           action: 'keep',
         }],
 
-        // Drop some high cardinality metrics.
+        // Keep limited set of metrics to reduce default usage, drop all others
         metric_relabel_configs: [
           {
             source_labels: ['__name__'],
-            regex: 'apiserver_admission_controller_admission_latencies_seconds_.*',
-            action: 'drop',
-          },
-          {
-            source_labels: ['__name__'],
-            regex: 'apiserver_admission_step_admission_latencies_seconds_.*',
-            action: 'drop',
+            regex: 'workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines',
+            action: 'keep',
           },
         ],
 

diff --git a/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet b/production/tanka/grafana-agent/v1/internal/kubernetes_instance.libsonnet
@@ -35,17 +35,12 @@ local k8s_tls_config(config) = {
           action: 'keep',
         }],
 
-        // Drop some high cardinality metrics.
+        // Keep limited set of metrics to reduce default usage, drop all others
         metric_relabel_configs: [
           {
             source_labels: ['__name__'],
-            regex: 'apiserver_admission_controller_admission_latencies_seconds_.*',
-            action: 'drop',
-          },
-          {
-            source_labels: ['__name__'],
-            regex: 'apiserver_admission_step_admission_latencies_seconds_.*',
-            action: 'drop',
+            regex: 'workqueue_queue_duration_seconds_bucket|process_cpu_seconds_total|process_resident_memory_bytes|workqueue_depth|rest_client_request_duration_seconds_bucket|workqueue_adds_total|up|rest_client_requests_total|apiserver_request_total|go_goroutines',
+            action: 'keep',
           },
         ],
       },