add nomad.agent.http.exceeded metric

Count of HTTP connections exceeding concurrency limit.
hashicorp · Jul 15, 2022 · 9768e42 · 9768e42
1 parent 3372c07
commit 9768e42
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/command/agent/http.go b/command/agent/http.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/armon/go-metrics"
 	"golang.org/x/time/rate"
 	"net"
 	"net/http"
@@ -278,6 +279,7 @@ func connLimiter(connLimit int, logger log.Logger) func(conn net.Conn, state htt
 		MaxConnsPerClientIP: connLimit,
 	}).HTTPConnStateFuncWithErrorHandler(func(err error, conn net.Conn) {
 		if err == connlimit.ErrPerClientIPLimitReached {
+			metrics.IncrCounter([]string{"nomad", "agent", "http", "exceeded"}, 1)
 			if n := limiter.Reserve(); n.Delay() == 0 {
 				logger.Warn("Too many concurrent connections", "address", conn.RemoteAddr().String(), "limit", connLimit)
 				conn.SetDeadline(time.Now().Add(10 * time.Millisecond))

diff --git a/website/content/docs/operations/metrics-reference.mdx b/website/content/docs/operations/metrics-reference.mdx
@@ -479,6 +479,14 @@ Raft database metrics are emitted by the `raft-boltdb` library.
 | `nomad.raft.boltdb.txstats.write`         | Count of total write operations           | Integer     | Counter |
 | `nomad.raft.boltdb.txstats.writeTime`     | Sample of write operation times           | Nanoseconds | Summary |
 
+## Agent Metrics
+
+Agent metrics are emitted by all Nomad agents running in either client or server mode.
+
+| Metric                                    | Description                                           | Unit        | Type    |
+| ----------------------------------------- | ----------------------------------------------------- | ----------- | ------- |
+| `nomad.agent.http.exceeded`               | Count of HTTP connections exceeding concurrency limit | Integer     | Counter |
+
 [tagged-metrics]: /docs/telemetry/metrics#tagged-metrics
 [s_port_plan_failure]: /s/port-plan-failure