From bb4ea56eefa16c5082e05f6ed9d78d0515930266 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 4 Nov 2019 13:35:45 +0000 Subject: [PATCH 1/3] Double the number of neighbours we gossip to, for improved coverage Previously a 3-node cluster would send to 1 peer each time, and a 7-node cluster to 2, so information could take a while to get everywhere. This calculation maintains the spirit that the number grows slowly in larger clusters - in a 100-node cluster each node will gossip to 12. --- gossip_test.go | 2 +- routes.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gossip_test.go b/gossip_test.go index bb597b7..d7ddd27 100644 --- a/gossip_test.go +++ b/gossip_test.go @@ -294,7 +294,7 @@ func TestRandomNeighbours(t *testing.T) { // Run randomNeighbours() several times, and count the distribution for trial := 0; trial < nTrials; trial++ { targets := r.randomNeighbours(ourself) - expected := int(math.Min(math.Log2(float64(test.nPeers)), float64(test.nNeighbours))) + expected := int(math.Min(2*math.Log2(float64(test.nPeers)), float64(test.nNeighbours))) require.Equal(t, expected, len(targets)) total += len(targets) for _, p := range targets { diff --git a/routes.go b/routes.go index 0df743c..05cbf29 100644 --- a/routes.go +++ b/routes.go @@ -128,7 +128,7 @@ func (r *routes) lookupOrCalculate(name PeerName, broadcast *broadcastRoutes, es return <-res } -// RandomNeighbours chooses min(log2(n_peers), n_neighbouring_peers) +// RandomNeighbours chooses min(2 log2(n_peers), n_neighbouring_peers) // neighbours, with a random distribution that is topology-sensitive, // favouring neighbours at the end of "bottleneck links". We determine the // latter based on the unicast routing table. If a neighbour appears as the @@ -152,7 +152,7 @@ func (r *routes) randomNeighbours(except PeerName) []PeerName { weights[dst]++ } } - needed := int(math.Min(math.Log2(float64(len(r.unicastAll))), float64(len(weights)))) + needed := int(math.Min(2*math.Log2(float64(len(r.unicastAll))), float64(len(weights)))) destinations := make([]PeerName, 0, needed) for len(destinations) < needed { // Pick a random point on the distribution and linear search for it From 68f49455f954d4249d7be2ae4f368ca48fa29f41 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 4 Nov 2019 13:40:20 +0000 Subject: [PATCH 2/3] Slow down accepting new connections, to reduce peak memory usage When 100 connections arrive at once, each one sends the whole topology and we read it all into memory at once then try to relay new information out again. These new parameters mean we will take 4 seconds to accept 100 connections, and 9 seconds to accept 200, so the system has time to process each message without spiking memory. --- router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router.go b/router.go index 0116a8e..9af5418 100644 --- a/router.go +++ b/router.go @@ -24,8 +24,8 @@ var ( const ( tcpHeartbeat = 30 * time.Second maxDuration = time.Duration(math.MaxInt64) - acceptMaxTokens = 100 - acceptTokenDelay = 100 * time.Millisecond // [2] + acceptMaxTokens = 20 + acceptTokenDelay = 50 * time.Millisecond ) // Config defines dimensions of configuration for the router. From 353a1b4d600401f091541da83df0113e3907378e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 4 Nov 2019 14:26:12 +0000 Subject: [PATCH 3/3] Update comment to match code --- routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes.go b/routes.go index 05cbf29..0e72d0f 100644 --- a/routes.go +++ b/routes.go @@ -136,7 +136,7 @@ func (r *routes) lookupOrCalculate(name PeerName, broadcast *broadcastRoutes, es // proportion of peers via that neighbour than other neighbours - then it is // chosen with a higher probability. // -// Note that we choose log2(n_peers) *neighbours*, not peers. Consequently, on +// Note that we choose 2log2(n_peers) *neighbours*, not peers. Consequently, on // sparsely connected peers this function returns a higher proportion of // neighbours than elsewhere. In extremis, on peers with fewer than // log2(n_peers) neighbours, all neighbours are returned.