Prometheus metrics

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## 0.7 (Unreleased)
 
 IMPROVEMENTS:
+ * api: Metrics endpoint exposes Prometheus formatted metrics [GH-3171]
  * telemetry: Add support for tagged metrics for Nomad clients [GH-3147]
 
 BUG FIXES:
@@ -15,7 +16,7 @@ BUG FIXES:
  * cli: Sort task groups when displaying a deployment [GH-3137]
  * cli: Handle reading files that are in a symlinked directory [GH-3164]
  * cli: All status commands handle even UUID prefixes with hyphens [GH-3122]
- * cli: Fix autocompletion of paths that include directories on zsh [GH-3129] 
+ * cli: Fix autocompletion of paths that include directories on zsh [GH-3129]
  * cli: Fix job deployment -latest handling of jobs without deployments
    [GH-3166]
  * cli: Hide CLI commands not expected to be run by user from autocomplete

command/agent/command.go

@@ -18,6 +18,7 @@ import (
 	metrics "github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/circonus"
 	"github.com/armon/go-metrics/datadog"
+	"github.com/armon/go-metrics/prometheus"
 	"github.com/hashicorp/consul/lib"
 	checkpoint "github.com/hashicorp/go-checkpoint"
 	gsyslog "github.com/hashicorp/go-syslog"
@@ -661,6 +662,15 @@ func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) {
 		fanout = append(fanout, sink)
 	}
 
+	// Configure the prometheus sink
+	if telConfig.PrometheusMetrics {
+		promSink, err := prometheus.NewPrometheusSink()
+		if err != nil {
+			return inm, err
+		}
+		fanout = append(fanout, promSink)
+	}
+
 	// Configure the datadog sink
 	if telConfig.DataDogAddr != "" {
 		sink, err := datadog.NewDogStatsdSink(telConfig.DataDogAddr, config.NodeName)

command/agent/config-test-fixtures/basic.hcl

@@ -92,6 +92,7 @@ acl {
 telemetry {
 	statsite_address = "127.0.0.1:1234"
 	statsd_address = "127.0.0.1:2345"
+	prometheus_metrics = true
 	disable_hostname = true
     collection_interval = "3s"
     publish_allocation_metrics = true

command/agent/config.go

@@ -356,6 +356,7 @@ type Telemetry struct {
 	StatsiteAddr             string        `mapstructure:"statsite_address"`
 	StatsdAddr               string        `mapstructure:"statsd_address"`
 	DataDogAddr              string        `mapstructure:"datadog_address"`
+	PrometheusMetrics        bool          `mapstructure:"prometheus_metrics"`
 	DisableHostname          bool          `mapstructure:"disable_hostname"`
 	UseNodeName              bool          `mapstructure:"use_node_name"`
 	CollectionInterval       string        `mapstructure:"collection_interval"`
@@ -1148,6 +1149,9 @@ func (a *Telemetry) Merge(b *Telemetry) *Telemetry {
 	if b.DataDogAddr != "" {
 		result.DataDogAddr = b.DataDogAddr
 	}
+	if b.PrometheusMetrics {
+		result.PrometheusMetrics = b.PrometheusMetrics
+	}
 	if b.DisableHostname {
 		result.DisableHostname = true
 	}

command/agent/config_parse.go

@@ -620,6 +620,7 @@ func parseTelemetry(result **Telemetry, list *ast.ObjectList) error {
 		"publish_allocation_metrics",
 		"publish_node_metrics",
 		"datadog_address",
+		"prometheus_metrics",
 		"circonus_api_token",
 		"circonus_api_app",
 		"circonus_api_url",

command/agent/config_parse_test.go

@@ -113,6 +113,7 @@ func TestConfig_Parse(t *testing.T) {
 				Telemetry: &Telemetry{
 					StatsiteAddr:               "127.0.0.1:1234",
 					StatsdAddr:                 "127.0.0.1:2345",
+					PrometheusMetrics:          true,
 					DisableHostname:            true,
 					UseNodeName:                false,
 					CollectionInterval:         "3s",

command/agent/config_test.go

@@ -54,6 +54,7 @@ func TestConfig_Merge(t *testing.T) {
 			StatsiteAddr:                       "127.0.0.1:8125",
 			StatsdAddr:                         "127.0.0.1:8125",
 			DataDogAddr:                        "127.0.0.1:8125",
+			PrometheusMetrics:                  true,
 			DisableHostname:                    false,
 			DisableTaggedMetrics:               true,
 			BackwardsCompatibleMetrics:         true,
@@ -182,6 +183,7 @@ func TestConfig_Merge(t *testing.T) {
 			StatsiteAddr:                       "127.0.0.2:8125",
 			StatsdAddr:                         "127.0.0.2:8125",
 			DataDogAddr:                        "127.0.0.1:8125",
+			PrometheusMetrics:                  true,
 			DisableHostname:                    true,
 			PublishNodeMetrics:                 true,
 			PublishAllocationMetrics:           true,

command/agent/http.go

@@ -80,8 +80,9 @@ func NewHTTPServer(agent *Agent, config *Config) (*HTTPServer, error) {
 	}
 	srv.registerHandlers(config.EnableDebug)
 
-	// Start the server
+	// Handle requests with gzip compression
 	go http.Serve(ln, gziphandler.GzipHandler(mux))
+
 	return srv, nil
 }
 
@@ -101,8 +102,9 @@ func newScadaHttp(agent *Agent, list net.Listener) *HTTPServer {
 	}
 	srv.registerHandlers(false) // Never allow debug for SCADA
 
-	// Start the server
+	// Handle requests with gzip compression
 	go http.Serve(list, gziphandler.GzipHandler(mux))
+
 	return srv
 }
 

command/agent/metrics_endpoint.go

@@ -2,15 +2,29 @@ package agent
 
 import (
 	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// MetricsRequest returns metrics for the agent. Metrics are JSON by default
+// but Prometheus is an optional format.
 func (s *HTTPServer) MetricsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
-	if req.Method == "GET" {
-		return s.newMetricsRequest(resp, req)
+	if req.Method != "GET" {
+		return nil, CodedError(405, ErrInvalidMethod)
+	}
+
+	if format := req.URL.Query().Get("format"); format == "prometheus" {
+		handlerOptions := promhttp.HandlerOpts{
+			ErrorLog:           s.logger,
+			ErrorHandling:      promhttp.ContinueOnError,
+			DisableCompression: true,
+		}
+
+		handler := promhttp.HandlerFor(prometheus.DefaultGatherer, handlerOptions)
+		handler.ServeHTTP(resp, req)
+		return nil, nil
 	}
-	return nil, CodedError(405, ErrInvalidMethod)
-}
 
-func (s *HTTPServer) newMetricsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	return s.agent.InmemSink.DisplayMetrics(resp, req)
 }