Use job name to differentiate beacon and validator metrics (ChainSafe…

…#4952) Grafana dashboards now use the "job" default label instead of "scrape_location" custom label to differentiate between beacon and validator metrics. The prometheus config files are updated to use the expected job name.
clionachee · Jan 12, 2023 · 0582f41 · 0582f41
1 parent 995d2a5
commit 0582f41
Show file tree

Hide file tree

Showing 7 changed files with 37 additions and 33 deletions.
diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json
@@ -2757,7 +2757,7 @@
               {
                 "matcher": {
                   "id": "byName",
-                  "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=\"beacon\", scrape_location=\"beacon\"}"
+                  "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=\"beacon\"}"
                 },
                 "properties": [
                   {
@@ -2993,7 +2993,7 @@
               {
                 "matcher": {
                   "id": "byName",
-                  "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}"
+                  "options": "{job=\"beacon\"}"
                 },
                 "properties": [
                   {

diff --git a/dashboards/lodestar_summary.json b/dashboards/lodestar_summary.json
@@ -575,7 +575,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "rate(process_cpu_user_seconds_total{scrape_location=\"beacon\"} [1m])",
+          "expr": "rate(process_cpu_user_seconds_total{job=\"beacon\"} [1m])",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -629,7 +629,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "process_heap_bytes{scrape_location=\"beacon\"}",
+          "expr": "process_heap_bytes{job=\"beacon\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -687,7 +687,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "process_start_time_seconds{scrape_location=\"beacon\"}*1000",
+          "expr": "process_start_time_seconds{job=\"beacon\"}*1000",
           "instant": false,
           "interval": "",
           "legendFormat": "",
@@ -815,7 +815,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "lodestar_version{scrape_location=\"beacon\"}",
+          "expr": "lodestar_version{job=\"beacon\"}",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -869,7 +869,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "lodestar_version{scrape_location=\"beacon\"}",
+          "expr": "lodestar_version{job=\"beacon\"}",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -932,7 +932,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "nodejs_version_info{scrape_location=\"beacon\"}",
+          "expr": "nodejs_version_info{job=\"beacon\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{version}}",
@@ -1553,7 +1553,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "sum(rate(nodejs_gc_duration_seconds_sum{scrape_location=\"beacon\"} [$rate_interval])) by (instance)",
+          "expr": "sum(rate(nodejs_gc_duration_seconds_sum{job=\"beacon\"} [$rate_interval])) by (instance)",
           "hide": false,
           "interval": "",
           "legendFormat": "gc_duration_sum",
@@ -1565,7 +1565,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "sum(rate(nodejs_gc_pause_seconds_total{scrape_location=\"beacon\"} [$rate_interval])) by (instance)",
+          "expr": "sum(rate(nodejs_gc_pause_seconds_total{job=\"beacon\"} [$rate_interval])) by (instance)",
           "hide": false,
           "interval": "",
           "legendFormat": "gc_pause_sum",
@@ -1687,7 +1687,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "avg_over_time( scrape_duration_seconds{scrape_location=\"beacon\"} [$rate_interval])",
+          "expr": "avg_over_time( scrape_duration_seconds{job=\"beacon\"} [$rate_interval])",
           "hide": false,
           "interval": "",
           "legendFormat": "prometheus_beacon_scrape_roundtrip",
@@ -1699,7 +1699,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "avg_over_time( scrape_duration_seconds{scrape_location=\"validator\"} [$rate_interval])",
+          "expr": "avg_over_time( scrape_duration_seconds{job=\"validator\"} [$rate_interval])",
           "hide": false,
           "interval": "",
           "legendFormat": "prometheus_validator_scrape_roundtrip",

diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json
@@ -261,7 +261,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "lodestar_version{scrape_location=\"validator\"}",
+          "expr": "lodestar_version{job=\"validator\"}",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -328,7 +328,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "nodejs_version_info{scrape_location=\"validator\"}",
+          "expr": "nodejs_version_info{job=\"validator\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{version}}",
@@ -393,7 +393,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "lodestar_version{scrape_location=\"validator\"}",
+          "expr": "lodestar_version{job=\"validator\"}",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -522,7 +522,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "process_heap_bytes{scrape_location=\"validator\"}",
+          "expr": "process_heap_bytes{job=\"validator\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -586,7 +586,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "nodejs_heap_size_used_bytes{scrape_location=\"validator\"}",
+          "expr": "nodejs_heap_size_used_bytes{job=\"validator\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"

diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json
@@ -220,8 +220,7 @@
               "group": false,
               "instance": true,
               "job": true,
-              "network": false,
-              "scrape_location": true
+              "network": false
             },
             "indexByName": {
               "Time": 1,
@@ -231,8 +230,7 @@
               "index": 0,
               "instance": 4,
               "job": 5,
-              "network": 6,
-              "scrape_location": 7
+              "network": 6
             },
             "renameByName": {}
           }

diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json
@@ -641,7 +641,7 @@
           {
             "matcher": {
               "id": "byName",
-              "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}"
+              "options": "{job=\"beacon\"}"
             },
             "properties": [
               {
@@ -2474,7 +2474,7 @@
               "expr": "avg_over_time(scrape_duration_seconds[$rate_interval])",
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{scrape_location}}",
+              "legendFormat": "{{job}}",
               "refId": "A"
             },
             {
@@ -2486,7 +2486,7 @@
               "expr": "rate(lodestar_metrics_scrape_seconds_sum[$rate_interval])\n/\nrate(lodestar_metrics_scrape_seconds_count[$rate_interval])",
               "hide": false,
               "interval": "",
-              "legendFormat": "{{scrape_location}} internal",
+              "legendFormat": "{{job}} internal",
               "refId": "B"
             }
           ],
@@ -2562,7 +2562,7 @@
               {
                 "matcher": {
                   "id": "byName",
-                  "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}"
+                  "options": "{job=\"beacon\"}"
                 },
                 "properties": [
                   {
@@ -2605,7 +2605,7 @@
               "exemplar": false,
               "expr": "1-up",
               "interval": "",
-              "legendFormat": "{{scrape_location}}",
+              "legendFormat": "{{job}}",
               "refId": "A"
             }
           ],

diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml
@@ -1,9 +1,15 @@
+global:
+  scrape_interval: 20s
+  scrape_timeout: 20s
+
+# Tags (values starting with #) have to be replaced in the Dockerfile with sed
+# Modified datasource to work with a network_mode: host
 scrape_configs:
-  - job_name: Lodestar
-    scrape_interval: 20s
-    scrape_timeout: 20s
+  - job_name: beacon
     metrics_path: /metrics
     static_configs:
-      # This tag is to be replaced in the Dockerfile with sed
-      # Modified datasource to work with a network_mode: host
-      - targets: ["#BEACON_URL", "#VC_URL"]
+      - targets: ["#BEACON_URL"]
+  - job_name: validator
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["#VC_URL"]
diff --git a/prometheus.yml b/prometheus.yml
@@ -1,5 +1,5 @@
 scrape_configs:
-  - job_name: Lodestar
+  - job_name: beacon
     scrape_interval: 5s
     metrics_path: /metrics
     static_configs: