From 1235885003412fc457308b222a4e7c86f984f5b0 Mon Sep 17 00:00:00 2001 From: sukun Date: Tue, 21 Feb 2023 18:04:43 +0530 Subject: [PATCH] add early muxer selection to swarm metrics --- core/network/conn.go | 2 + dashboards/swarm/swarm.json | 135 ++++++++++++++++++++++++++- p2p/net/swarm/swarm_metrics.go | 14 ++- p2p/net/swarm/swarm_metrics_test.go | 4 +- p2p/net/upgrader/conn.go | 12 ++- p2p/net/upgrader/upgrader.go | 17 ++-- p2p/security/noise/session.go | 1 + p2p/security/noise/transport_test.go | 2 + p2p/security/tls/transport.go | 15 +-- p2p/security/tls/transport_test.go | 1 + 10 files changed, 177 insertions(+), 26 deletions(-) diff --git a/core/network/conn.go b/core/network/conn.go index 4191a9d544..6fac0784ab 100644 --- a/core/network/conn.go +++ b/core/network/conn.go @@ -43,6 +43,8 @@ type ConnectionState struct { Security protocol.ID // the transport used on this connection. For example: tcp Transport string + // indicates whether StreamMultiplexer was selected using inlined muxer negotiation + EarlyMuxerSelection bool } // ConnSecurity is the interface that one can mix into a connection interface to diff --git a/dashboards/swarm/swarm.json b/dashboards/swarm/swarm.json index 09eee4ced7..59044b9764 100644 --- a/dashboards/swarm/swarm.json +++ b/dashboards/swarm/swarm.json @@ -1059,6 +1059,139 @@ "title": "Connection Duration", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "early muxer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "regular" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(libp2p_swarm_handshake_latency_seconds_sum{transport=\"tcp\",early_muxer=\"true\"}[$__rate_interval])) / sum(rate(libp2p_swarm_handshake_latency_seconds_count{transport=\"tcp\",early_muxer=\"true\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "early muxer", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(libp2p_swarm_handshake_latency_seconds_sum{transport=\"tcp\",early_muxer=\"false\"}[$__rate_interval])) / sum(rate(libp2p_swarm_handshake_latency_seconds_count{transport=\"tcp\",early_muxer=\"false\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "regular", + "range": true, + "refId": "B" + } + ], + "title": "Handshake Latency (Early Muxer Selection)", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { @@ -2485,6 +2618,6 @@ "timezone": "", "title": "libp2p Swarm", "uid": "a15PyhO4z", - "version": 12, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/p2p/net/swarm/swarm_metrics.go b/p2p/net/swarm/swarm_metrics.go index 6af2cf919b..5fbc9b4f45 100644 --- a/p2p/net/swarm/swarm_metrics.go +++ b/p2p/net/swarm/swarm_metrics.go @@ -26,7 +26,7 @@ var ( Name: "connections_opened_total", Help: "Connections Opened", }, - []string{"dir", "transport", "security", "muxer", "ip_version"}, + []string{"dir", "transport", "security", "muxer", "early_muxer", "ip_version"}, ) keyTypes = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -42,7 +42,7 @@ var ( Name: "connections_closed_total", Help: "Connections Closed", }, - []string{"dir", "transport", "security", "muxer", "ip_version"}, + []string{"dir", "transport", "security", "muxer", "early_muxer", "ip_version"}, ) dialError = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -59,7 +59,7 @@ var ( Help: "Duration of a Connection", Buckets: prometheus.ExponentialBuckets(1.0/16, 2, 25), // up to 24 days }, - []string{"dir", "transport", "security", "muxer", "ip_version"}, + []string{"dir", "transport", "security", "muxer", "early_muxer", "ip_version"}, ) connHandshakeLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -68,7 +68,7 @@ var ( Help: "Duration of the libp2p Handshake", Buckets: prometheus.ExponentialBuckets(0.001, 1.3, 35), }, - []string{"transport", "security", "muxer", "ip_version"}, + []string{"transport", "security", "muxer", "early_muxer", "ip_version"}, ) ) @@ -105,6 +105,12 @@ func appendConnectionState(tags []string, cs network.ConnectionState) []string { // For example, QUIC doesn't set security nor muxer. tags = append(tags, string(cs.Security)) tags = append(tags, string(cs.StreamMultiplexer)) + + earlyMuxer := "false" + if cs.EarlyMuxerSelection { + earlyMuxer = "true" + } + tags = append(tags, earlyMuxer) return tags } diff --git a/p2p/net/swarm/swarm_metrics_test.go b/p2p/net/swarm/swarm_metrics_test.go index de159217d8..0ad3bea5a3 100644 --- a/p2p/net/swarm/swarm_metrics_test.go +++ b/p2p/net/swarm/swarm_metrics_test.go @@ -50,8 +50,8 @@ func TestMetricsNoAllocNoCover(t *testing.T) { mt := NewMetricsTracer() connections := []network.ConnectionState{ - {StreamMultiplexer: "yamux", Security: "tls", Transport: "tcp"}, - {StreamMultiplexer: "yamux", Security: "noise", Transport: "tcp"}, + {StreamMultiplexer: "yamux", Security: "tls", Transport: "tcp", EarlyMuxerSelection: true}, + {StreamMultiplexer: "yamux", Security: "noise", Transport: "tcp", EarlyMuxerSelection: false}, {StreamMultiplexer: "", Security: "", Transport: "quic"}, {StreamMultiplexer: "mplex", Security: "noise", Transport: "tcp"}, } diff --git a/p2p/net/upgrader/conn.go b/p2p/net/upgrader/conn.go index 5db2175517..3a94c499cc 100644 --- a/p2p/net/upgrader/conn.go +++ b/p2p/net/upgrader/conn.go @@ -16,8 +16,9 @@ type transportConn struct { scope network.ConnManagementScope stat network.ConnStats - muxer protocol.ID - security protocol.ID + muxer protocol.ID + security protocol.ID + earlyMuxerSelection bool } var _ transport.CapableConn = &transportConn{} @@ -56,8 +57,9 @@ func (t *transportConn) Close() error { func (t *transportConn) ConnState() network.ConnectionState { return network.ConnectionState{ - StreamMultiplexer: t.muxer, - Security: t.security, - Transport: "tcp", + StreamMultiplexer: t.muxer, + Security: t.security, + Transport: "tcp", + EarlyMuxerSelection: t.earlyMuxerSelection, } } diff --git a/p2p/net/upgrader/upgrader.go b/p2p/net/upgrader/upgrader.go index 658e5df7b2..a908b15497 100644 --- a/p2p/net/upgrader/upgrader.go +++ b/p2p/net/upgrader/upgrader.go @@ -186,14 +186,15 @@ func (u *upgrader) upgrade(ctx context.Context, t transport.Transport, maconn ma } tc := &transportConn{ - MuxedConn: smconn, - ConnMultiaddrs: maconn, - ConnSecurity: sconn, - transport: t, - stat: stat, - scope: connScope, - muxer: muxer, - security: security, + MuxedConn: smconn, + ConnMultiaddrs: maconn, + ConnSecurity: sconn, + transport: t, + stat: stat, + scope: connScope, + muxer: muxer, + security: security, + earlyMuxerSelection: sconn.ConnState().EarlyMuxerSelection, } return tc, nil } diff --git a/p2p/security/noise/session.go b/p2p/security/noise/session.go index 803ed7cba3..faa4fc7454 100644 --- a/p2p/security/noise/session.go +++ b/p2p/security/noise/session.go @@ -138,6 +138,7 @@ func (s *secureSession) Close() error { func SessionWithConnState(s *secureSession, muxer protocol.ID) *secureSession { if s != nil { s.connectionState.StreamMultiplexer = muxer + s.connectionState.EarlyMuxerSelection = muxer != "" } return s } diff --git a/p2p/security/noise/transport_test.go b/p2p/security/noise/transport_test.go index 7cdd562942..8be07a10ae 100644 --- a/p2p/security/noise/transport_test.go +++ b/p2p/security/noise/transport_test.go @@ -686,7 +686,9 @@ func TestHandshakeWithTransportEarlyData(t *testing.T) { defer respConn.Close() require.Equal(t, expectedProto, initConn.connectionState.StreamMultiplexer) + require.Equal(t, expectedProto != "", initConn.connectionState.EarlyMuxerSelection) require.Equal(t, expectedProto, respConn.connectionState.StreamMultiplexer) + require.Equal(t, expectedProto != "", respConn.connectionState.EarlyMuxerSelection) initData := []byte("Test data for noise transport") _, err := initConn.Write(initData) diff --git a/p2p/security/tls/transport.go b/p2p/security/tls/transport.go index 3ca837e5ef..b3b7b84807 100644 --- a/p2p/security/tls/transport.go +++ b/p2p/security/tls/transport.go @@ -166,12 +166,15 @@ func (t *Transport) setupConn(tlsConn *tls.Conn, remotePubKey ci.PubKey) (sec.Se } return &conn{ - Conn: tlsConn, - localPeer: t.localPeer, - privKey: t.privKey, - remotePeer: remotePeerID, - remotePubKey: remotePubKey, - connectionState: network.ConnectionState{StreamMultiplexer: protocol.ID(nextProto)}, + Conn: tlsConn, + localPeer: t.localPeer, + privKey: t.privKey, + remotePeer: remotePeerID, + remotePubKey: remotePubKey, + connectionState: network.ConnectionState{ + StreamMultiplexer: protocol.ID(nextProto), + EarlyMuxerSelection: nextProto != "", + }, }, nil } diff --git a/p2p/security/tls/transport_test.go b/p2p/security/tls/transport_test.go index ab7cb5f382..206c14ada6 100644 --- a/p2p/security/tls/transport_test.go +++ b/p2p/security/tls/transport_test.go @@ -256,6 +256,7 @@ func TestHandshakeWithNextProtoSucceeds(t *testing.T) { require.True(t, clientConn.RemotePublicKey().Equals(serverKey.GetPublic()), "server public key mismatch") require.True(t, serverConn.RemotePublicKey().Equals(clientKey.GetPublic()), "client public key mismatch") require.Equal(t, clientConn.ConnState().StreamMultiplexer, expectedMuxer) + require.Equal(t, clientConn.ConnState().EarlyMuxerSelection, expectedMuxer != "") // exchange some data _, err = serverConn.Write([]byte("foobar")) require.NoError(t, err)