fix prometheus endpoint

hashicorp · Sep 8, 2017 · 73b05df · 73b05df
1 parent 7d751cc
commit 73b05df
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 11 deletions.
diff --git a/command/agent/command.go b/command/agent/command.go
@@ -18,6 +18,7 @@ import (
 	metrics "github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/circonus"
 	"github.com/armon/go-metrics/datadog"
+	"github.com/armon/go-metrics/prometheus"
 	"github.com/hashicorp/consul/lib"
 	checkpoint "github.com/hashicorp/go-checkpoint"
 	gsyslog "github.com/hashicorp/go-syslog"
@@ -661,6 +662,12 @@ func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) {
 		fanout = append(fanout, sink)
 	}
 
+	// Configure the prometheus sink
+	promSink, err := prometheus.NewPrometheusSink()
+	if err != nil {
+		fanout = append(fanout, promSink)
+	}
+
 	// Configure the datadog sink
 	if telConfig.DataDogAddr != "" {
 		sink, err := datadog.NewDogStatsdSink(telConfig.DataDogAddr, config.NodeName)

diff --git a/command/agent/http.go b/command/agent/http.go
@@ -15,6 +15,7 @@ import (
 	"github.com/NYTimes/gziphandler"
 	"github.com/hashicorp/nomad/helper/tlsutil"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/ugorji/go/codec"
 )
@@ -81,8 +82,15 @@ func NewHTTPServer(agent *Agent, config *Config) (*HTTPServer, error) {
 	}
 	srv.registerHandlers(config.EnableDebug)
 
-	// Start the server
+	// Handle requests with gzip compression
 	go http.Serve(ln, gziphandler.GzipHandler(mux))
+
+	// Register Prometheus separately as this requires uncompressed responses
+	srv.registerPrometheusHandler()
+
+	// Handle requests without gzip compression
+	go http.Serve(ln, mux)
+
 	return srv, nil
 }
 
@@ -102,8 +110,15 @@ func newScadaHttp(agent *Agent, list net.Listener) *HTTPServer {
 	}
 	srv.registerHandlers(false) // Never allow debug for SCADA
 
-	// Start the server
+	// Handle requests with gzip compression
 	go http.Serve(list, gziphandler.GzipHandler(mux))
+
+	// Register Prometheus separately as this requires uncompressed responses
+	srv.registerPrometheusHandler()
+
+	// Handle requests without gzip compression
+	go http.Serve(list, mux)
+
 	return srv
 }
 
@@ -132,6 +147,19 @@ func (s *HTTPServer) Shutdown() {
 	}
 }
 
+// Create a prometheus handler and register it separately, as other requests
+// use gzip compression
+func (s *HTTPServer) registerPrometheusHandler() {
+	handlerOptions := promhttp.HandlerOpts{
+		ErrorLog:           s.logger,
+		ErrorHandling:      promhttp.ContinueOnError,
+		DisableCompression: true,
+	}
+
+	handler := promhttp.HandlerFor(prometheus.DefaultGatherer, handlerOptions)
+	s.mux.Handle("/metrics", handler)
+}
+
 // registerHandlers is used to attach our handlers to the mux
 func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest))
@@ -170,7 +198,6 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/agent/keyring/", s.wrap(s.KeyringOperationRequest))
 
 	s.mux.HandleFunc("/v1/metrics", s.wrap(s.MetricsRequest))
-	s.mux.Handle("/v1/prometheus-metrics", promhttp.Handler())
 
 	s.mux.HandleFunc("/v1/validate/job", s.wrap(s.ValidateJobRequest))
 

diff --git a/vendor/vendor.json b/vendor/vendor.json
@@ -86,6 +86,12 @@
 			"revision": "023a4bbe4bb9bfb23ee7e1afc8d0abad217641f3",
 			"revisionTime": "2017-08-09T01:16:44Z"
 		},
+		{
+			"checksumSHA1": "sLK17mCfroZDH/jcaaEEodDWwjw=",
+			"path": "github.com/armon/go-metrics/prometheus",
+			"revision": "023a4bbe4bb9bfb23ee7e1afc8d0abad217641f3",
+			"revisionTime": "2017-08-09T01:16:44Z"
+		},
 		{
 			"checksumSHA1": "gNO0JNpLzYOdInGeq7HqMZUzx9M=",
 			"path": "github.com/armon/go-radix",
@@ -391,14 +397,14 @@
 		{
 			"checksumSHA1": "tdhmIGUaoOMEDymMC23qTS7bt0g=",
 			"path": "github.com/docker/docker/pkg/ioutils",
-			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
-			"revisionTime": "2016-05-20T15:17:10Z"
+			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
+			"revisionTime": "2016-05-28T08:11:04Z"
 		},
 		{
 			"checksumSHA1": "tdhmIGUaoOMEDymMC23qTS7bt0g=",
 			"path": "github.com/docker/docker/pkg/ioutils",
-			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
-			"revisionTime": "2016-05-28T08:11:04Z"
+			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
+			"revisionTime": "2016-05-20T15:17:10Z"
 		},
 		{
 			"checksumSHA1": "BlFSSK7zUjPzPuxkLmM/0wpvku8=",
@@ -421,14 +427,14 @@
 		{
 			"checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=",
 			"path": "github.com/docker/docker/pkg/pools",
-			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
-			"revisionTime": "2016-05-28T08:11:04Z"
+			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
+			"revisionTime": "2016-05-20T15:17:10Z"
 		},
 		{
 			"checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=",
 			"path": "github.com/docker/docker/pkg/pools",
-			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
-			"revisionTime": "2016-05-20T15:17:10Z"
+			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
+			"revisionTime": "2016-05-28T08:11:04Z"
 		},
 		{
 			"checksumSHA1": "txf3EORYff4hO6PEvwBm2lyh1MU=",