Merge pull request #22803 from travisdowns/td-CORE-6827-rate-rejected…

…-connections-metric Add rate-rejected connection metric
redpanda-data · Aug 9, 2024 · 0495334 · 0495334
2 parents f26baa2 + b1f3372
commit 0495334
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 15 deletions.
diff --git a/src/v/net/include/net/server_probe.h b/src/v/net/include/net/server_probe.h
@@ -38,7 +38,13 @@ class server_probe {
 
     void connection_close_error() { ++_connection_close_error; }
 
-    void connection_rejected() { ++_connections_rejected; }
+    void connection_rejected_open_limit() {
+        ++_connections_rejected_open_limit;
+    }
+
+    void connection_rejected_rate_limit() {
+        ++_connections_rejected_rate_limit;
+    }
 
     void add_bytes_sent(size_t sent) { _out_bytes += sent; }
 
@@ -56,8 +62,6 @@ class server_probe {
 
     void waiting_for_available_memory() { ++_requests_blocked_memory; }
 
-    void timeout_waiting_rate_limit() { ++_declined_new_connections; }
-
     void waiting_for_conection_rate() { ++_connections_wait_rate; }
 
     // metric used to signal a produce request with a timestamp too far into the
@@ -85,11 +89,14 @@ class server_probe {
     uint64_t _service_errors = 0;
     uint32_t _connections = 0;
     uint32_t _connection_close_error = 0;
-    uint64_t _connections_rejected = 0;
+    // connections rejected as we hit our "open connections" limit
+    uint64_t _connections_rejected_open_limit = 0;
+    // connections rejected as we hit our connection rate limit and
+    // delaying the connection was not sufficient to stay under the limit
+    uint32_t _connections_rejected_rate_limit = 0;
     uint32_t _corrupted_headers = 0;
     uint32_t _method_not_found_errors = 0;
     uint32_t _requests_blocked_memory = 0;
-    uint32_t _declined_new_connections = 0;
     uint32_t _connections_wait_rate = 0;
     uint32_t _produce_bad_create_time = 0;
     friend std::ostream& operator<<(std::ostream& o, const server_probe& p);

diff --git a/src/v/net/probes.cc b/src/v/net/probes.cc
@@ -58,9 +58,17 @@ void server_probe::setup_metrics(
             "{}: Number of errors when shutting down the connection", proto))),
         sm::make_counter(
           "connections_rejected",
-          [this] { return _connections_rejected; },
+          [this] { return _connections_rejected_open_limit; },
           sm::description(ssx::sformat(
-            "{}: Number of connections rejected for hitting connection limits",
+            "{}: Number of connection attempts rejected for hitting open "
+            "connection count limits",
+            proto))),
+        sm::make_counter(
+          "connections_rejected_rate_limit",
+          [this] { return _connections_rejected_rate_limit; },
+          sm::description(ssx::sformat(
+            "{}: Number of connection attempts rejected for hitting "
+            "connection rate limits",
             proto))),
         sm::make_counter(
           "requests_completed",
@@ -166,7 +174,10 @@ std::ostream& operator<<(std::ostream& o, const server_probe& p) {
       << "connects: " << p._connects << ", "
       << "current connections: " << p._connections << ", "
       << "connection close errors: " << p._connection_close_error << ", "
-      << "connections rejected: " << p._connections_rejected << ", "
+      << "connections rejected (open limit): "
+      << p._connections_rejected_open_limit << ", "
+      << "connections rejected (rate limit): "
+      << p._connections_rejected_rate_limit << ", "
       << "requests received: " << p._requests_received << ", "
       << "requests completed: " << p._requests_completed << ", "
       << "service errors: " << p._service_errors << ", "
@@ -175,7 +186,6 @@ std::ostream& operator<<(std::ostream& o, const server_probe& p) {
       << "corrupted headers: " << p._corrupted_headers << ", "
       << "method not found errors: " << p._method_not_found_errors << ", "
       << "requests blocked by memory: " << p._requests_blocked_memory << ", "
-      << "declined new connections: " << p._declined_new_connections << ", "
       << "connections wait rate: " << p._connections_wait_rate << ", "
       << "produce bad create time: " << p._produce_bad_create_time << ", "
       << "}";

diff --git a/src/v/net/server.cc b/src/v/net/server.cc
@@ -218,11 +218,11 @@ ss::future<ss::stop_iteration> server::accept_finish(
           ar.remote_address.addr());
         if (!cq_units.live()) {
             // Connection limit hit, drop this connection.
-            _probe->connection_rejected();
+            _probe->connection_rejected_open_limit();
             vlog(
               _log.warn,
-              "Connection limit reached, rejecting {}",
-              ar.remote_address.addr());
+              "Open connection limit reached, rejecting {}",
+              ar.remote_address);
             co_return ss::stop_iteration::no;
         }
     }
@@ -248,10 +248,10 @@ ss::future<ss::stop_iteration> server::accept_finish(
         } catch (const std::exception& e) {
             vlog(
               _log.trace,
-              "Timeout while waiting free token for connection rate. "
-              "addr:{}",
+              "Connection rate limit reached and no token available after "
+              "wait, rejecting {}",
               ar.remote_address);
-            _probe->timeout_waiting_rate_limit();
+            _probe->connection_rejected_rate_limit();
             co_return ss::stop_iteration::no;
         }
     }