Add HTTP instrumentation for requests in flight

Note that the api/v2 calls are currently not instrumented at all. This
adds a TODO note.

Signed-off-by: beorn7 <beorn@soundcloud.com>

api/api.go

@@ -101,6 +101,9 @@ func (api *API) Register(
 	if routePrefix != "/" {
 		apiPrefix = routePrefix
 	}
+	// TODO(beorn7): HTTP instrumentation is only in place for Router. Since
+	// /api/v2 works on the Handler level, it is currently not instrumented
+	// at all!
 	mux.Handle(
 		apiPrefix+"/api/v2/",
 		limiter(http.StripPrefix(apiPrefix+"/api/v2", api.v2.Handler)),

cmd/alertmanager/main.go

@@ -77,6 +77,13 @@ var (
 		},
 		[]string{"handler", "method"},
 	)
+	requestsInFlight = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "alertmanager_http_requests_in_flight",
+			Help: "Current number of HTTP requests being processed.",
+		},
+		[]string{"handler", "method"},
+	)
 	responseSize = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "alertmanager_http_response_size_bytes",
@@ -98,11 +105,15 @@ func init() {
 }
 
 func instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
+	handlerLabel := prometheus.Labels{"handler": handlerName}
 	return promhttp.InstrumentHandlerDuration(
-		requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}),
+		requestDuration.MustCurryWith(handlerLabel),
 		promhttp.InstrumentHandlerResponseSize(
-			responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}),
-			handler,
+			responseSize.MustCurryWith(handlerLabel),
+			promhttp.InstrumentHandlerInFlight(
+				requestsInFlight.MustCurryWith(handlerLabel),
+				handler,
+			),
 		),
 	)
 }