From 9768e429ea457a5f3b39cb3d330884de5a094adb Mon Sep 17 00:00:00 2001 From: Will Jordan Date: Fri, 15 Jul 2022 16:34:55 -0700 Subject: [PATCH] add nomad.agent.http.exceeded metric Count of HTTP connections exceeding concurrency limit. --- command/agent/http.go | 2 ++ website/content/docs/operations/metrics-reference.mdx | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/command/agent/http.go b/command/agent/http.go index 84c65992bf65..00d9af28fa5c 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/armon/go-metrics" "golang.org/x/time/rate" "net" "net/http" @@ -278,6 +279,7 @@ func connLimiter(connLimit int, logger log.Logger) func(conn net.Conn, state htt MaxConnsPerClientIP: connLimit, }).HTTPConnStateFuncWithErrorHandler(func(err error, conn net.Conn) { if err == connlimit.ErrPerClientIPLimitReached { + metrics.IncrCounter([]string{"nomad", "agent", "http", "exceeded"}, 1) if n := limiter.Reserve(); n.Delay() == 0 { logger.Warn("Too many concurrent connections", "address", conn.RemoteAddr().String(), "limit", connLimit) conn.SetDeadline(time.Now().Add(10 * time.Millisecond)) diff --git a/website/content/docs/operations/metrics-reference.mdx b/website/content/docs/operations/metrics-reference.mdx index edc1286a8578..89a80a7e9f2b 100644 --- a/website/content/docs/operations/metrics-reference.mdx +++ b/website/content/docs/operations/metrics-reference.mdx @@ -479,6 +479,14 @@ Raft database metrics are emitted by the `raft-boltdb` library. | `nomad.raft.boltdb.txstats.write` | Count of total write operations | Integer | Counter | | `nomad.raft.boltdb.txstats.writeTime` | Sample of write operation times | Nanoseconds | Summary | +## Agent Metrics + +Agent metrics are emitted by all Nomad agents running in either client or server mode. + +| Metric | Description | Unit | Type | +| ----------------------------------------- | ----------------------------------------------------- | ----------- | ------- | +| `nomad.agent.http.exceeded` | Count of HTTP connections exceeding concurrency limit | Integer | Counter | + [tagged-metrics]: /docs/telemetry/metrics#tagged-metrics [s_port_plan_failure]: /s/port-plan-failure