From 7fea04652567eceb4703b7dcc7a5adb91d31ee13 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 12 Sep 2017 01:03:10 +0000 Subject: [PATCH 1/8] add prometheus metrics for configuration with consul and nomad --- integrations/prometheus/prometheus.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 integrations/prometheus/prometheus.yml diff --git a/integrations/prometheus/prometheus.yml b/integrations/prometheus/prometheus.yml new file mode 100644 index 000000000000..f4b47508e359 --- /dev/null +++ b/integrations/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +global: + external_labels: + monitor: 'codelab-monitor' + +scrape_configs: + - job_name: 'nomad_data' + + consul_sd_configs: + - server: 'localhost:8500' + services: ['nomad-client', 'nomad-server-nomad'] + + scrape_interval: 5s + metrics_path: /metrics + From 401f3f90076d0b1beeade2e2b508d779124877d3 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 12 Sep 2017 01:26:27 +0000 Subject: [PATCH 2/8] Readme instructions --- integrations/prometheus/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 integrations/prometheus/README.md diff --git a/integrations/prometheus/README.md b/integrations/prometheus/README.md new file mode 100644 index 000000000000..2be179970259 --- /dev/null +++ b/integrations/prometheus/README.md @@ -0,0 +1,11 @@ +# Prometheus Metrics + +This configuration file can be used to set up a Prometheus instance to read +data from Nomad cluster, using Consul for service discovery. + +Requirements: + - See Prometheus's + [Getting Started](https://prometheus.io/docs/introduction/getting_started/) + guide for instructions on how to set up a Prometheus server. + - A running Consul server, which will need to be added to this configuration + file. From 2c53439ef5b56284ee24c3fa7349e5e801bfcc59 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 12 Sep 2017 17:05:45 +0000 Subject: [PATCH 3/8] update consul information --- integrations/prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/prometheus/prometheus.yml b/integrations/prometheus/prometheus.yml index f4b47508e359..80ce39c94845 100644 --- a/integrations/prometheus/prometheus.yml +++ b/integrations/prometheus/prometheus.yml @@ -7,7 +7,7 @@ scrape_configs: consul_sd_configs: - server: 'localhost:8500' - services: ['nomad-client', 'nomad-server-nomad'] + services: ['nomad-client', 'nomad'] scrape_interval: 5s metrics_path: /metrics From 2783b27c4ce58764519099c7dcf065ee2ff3b9fc Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Wed, 13 Sep 2017 14:28:34 +0000 Subject: [PATCH 4/8] filter nomad services for only http --- integrations/prometheus/prometheus.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integrations/prometheus/prometheus.yml b/integrations/prometheus/prometheus.yml index 80ce39c94845..d0c336d559c6 100644 --- a/integrations/prometheus/prometheus.yml +++ b/integrations/prometheus/prometheus.yml @@ -9,6 +9,11 @@ scrape_configs: - server: 'localhost:8500' services: ['nomad-client', 'nomad'] + relabel_configs: + - source_labels: ['__meta_consul_tags'] + regex: '(.*)http(.*)' + action: keep + scrape_interval: 5s metrics_path: /metrics From 66c0df7332aa8b0caf4a2661c98015ca7c6867c7 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Wed, 13 Sep 2017 14:31:01 +0000 Subject: [PATCH 5/8] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eade1e4a8738..4a37d00feb80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ IMPROVEMENTS: * api: Metrics endpoint exposes Prometheus formatted metrics [GH-3171] * telemetry: Add support for tagged metrics for Nomad clients [GH-3147] + * telemetry: Add basic Prometheus configuration for a Nomad cluster [GH-3186] BUG FIXES: * core: *Fix restoration of stopped periodic jobs [GH-3201] From 7212566f5cd39bbceb868d703355955f7d592348 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Thu, 14 Sep 2017 18:11:47 +0000 Subject: [PATCH 6/8] update metrics path --- integrations/prometheus/prometheus.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrations/prometheus/prometheus.yml b/integrations/prometheus/prometheus.yml index d0c336d559c6..f4b3a05c6cdd 100644 --- a/integrations/prometheus/prometheus.yml +++ b/integrations/prometheus/prometheus.yml @@ -15,5 +15,8 @@ scrape_configs: action: keep scrape_interval: 5s - metrics_path: /metrics + metrics_path: /v1/metrics + params: + format: ['prometheus'] + From 3a8320caa2f031bc550d6fe02a4f47f1d2f6d497 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 15 Sep 2017 03:20:28 +0000 Subject: [PATCH 7/8] fixups from code review --- integrations/prometheus/README.md | 5 +++-- integrations/prometheus/prometheus.yml | 8 +------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/integrations/prometheus/README.md b/integrations/prometheus/README.md index 2be179970259..674010cb753f 100644 --- a/integrations/prometheus/README.md +++ b/integrations/prometheus/README.md @@ -7,5 +7,6 @@ Requirements: - See Prometheus's [Getting Started](https://prometheus.io/docs/introduction/getting_started/) guide for instructions on how to set up a Prometheus server. - - A running Consul server, which will need to be added to this configuration - file. + - A running Consul server. This configuration is written for a local remote + Consul agent, and will need to be updated in the case Consul agent is on a + remote server. diff --git a/integrations/prometheus/prometheus.yml b/integrations/prometheus/prometheus.yml index f4b3a05c6cdd..0e644c5f27d5 100644 --- a/integrations/prometheus/prometheus.yml +++ b/integrations/prometheus/prometheus.yml @@ -1,9 +1,5 @@ -global: - external_labels: - monitor: 'codelab-monitor' - scrape_configs: - - job_name: 'nomad_data' + - job_name: 'nomad_metrics' consul_sd_configs: - server: 'localhost:8500' @@ -18,5 +14,3 @@ scrape_configs: metrics_path: /v1/metrics params: format: ['prometheus'] - - From e7933df1257d859e970994505ffd4fbcabe10e08 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 15 Sep 2017 18:24:17 +0000 Subject: [PATCH 8/8] reworded prometheus readme --- integrations/prometheus/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/prometheus/README.md b/integrations/prometheus/README.md index 674010cb753f..5a4348266948 100644 --- a/integrations/prometheus/README.md +++ b/integrations/prometheus/README.md @@ -7,6 +7,6 @@ Requirements: - See Prometheus's [Getting Started](https://prometheus.io/docs/introduction/getting_started/) guide for instructions on how to set up a Prometheus server. - - A running Consul server. This configuration is written for a local remote - Consul agent, and will need to be updated in the case Consul agent is on a - remote server. + - This configuration is written assuming there is a local Consul agent being + run along side the Prometheus server. If this is not the case, the + configuration will need to be updated to point at the remote Consul agent.