feat: track leaked libp2p connections (#6656)

* feat: track leaked libp2p connections * update var name --------- Co-authored-by: Nico Flaig <nflaig@protonmail.com>
ChainSafe · Apr 11, 2024 · f21660e · f21660e · github-actions · Apr 11, 2024
1 parent e029851
commit f21660e
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts
@@ -112,6 +112,10 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) {
         buckets: [0.001, 0.01, 0.1, 1],
       }),
     },
+    leakedConnectionsCount: register.gauge({
+      name: "lodestar_peer_manager_leaked_connections_count",
+      help: "Total libp2p leaked connections detected by lodestar",
+    }),
 
     discovery: {
       peersToConnect: register.gauge({

diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts
@@ -504,11 +504,12 @@ export class PeerManager {
 
     // Prune connectedPeers map in case it leaks. It has happen in previous nodes,
     // disconnect is not always called for all peers
-    if (this.connectedPeers.size > connectedPeers.length * 2) {
+    if (this.connectedPeers.size > connectedPeers.length * 1.1) {
       const actualConnectedPeerIds = new Set(connectedPeers.map((peerId) => peerId.toString()));
       for (const peerIdStr of this.connectedPeers.keys()) {
         if (!actualConnectedPeerIds.has(peerIdStr)) {
           this.connectedPeers.delete(peerIdStr);
+          this.metrics?.leakedConnectionsCount.inc();
         }
       }
     }
Benchmark suite	Current: `f21660e`	Previous: `e029851`	Ratio
getExpectedWithdrawals 250000 eb:0.95,eth1:0.3,we:0.05,wn:0,smpl:42	26.873 us/op	8.2670 us/op	3.25
getExpectedWithdrawals 250000 eb:0,eth1:0,we:0,wn:0,smpl:16384	2.8746 ms/op	798.40 us/op	3.60