From f596b01fc1dab211c898244151017867d182909d Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Fri, 28 Jun 2019 08:04:55 +0200
Subject: [PATCH] feat: randomly pick preload node (#2194)

This PR updates the preloader to randomly pick a node when preloading data.

It actually shuffles the list of _available_ preload nodes for every preload
request, and works its way through. This helps spread the load between available
preload nodes.

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>
---
 package.json                           |  1 +
 src/core/preload.js                    |  3 +-
 test/core/bitswap.spec.js              |  3 +-
 test/core/block.spec.js                |  3 +-
 test/core/bootstrap.spec.js            |  3 +-
 test/core/circuit-relay.js             |  3 +-
 test/core/create-node.spec.js          | 42 +++++++++++++++++---------
 test/core/dag.spec.js                  |  3 +-
 test/core/dht.spec.js                  |  3 +-
 test/core/files-sharding.spec.js       |  6 ++--
 test/core/files.spec.js                |  3 +-
 test/core/init.spec.js                 |  3 +-
 test/core/kad-dht.node.js              |  3 +-
 test/core/key-exchange.js              |  3 +-
 test/core/name-pubsub.js               |  3 +-
 test/core/name.spec.js                 |  9 ++++--
 test/core/object.spec.js               |  3 +-
 test/core/pin-set.js                   |  3 +-
 test/core/pin.js                       |  3 +-
 test/core/pin.spec.js                  |  3 +-
 test/core/ping.spec.js                 |  3 +-
 test/core/stats.spec.js                |  3 +-
 test/core/swarm.spec.js                |  3 +-
 test/core/utils.js                     |  3 +-
 test/gateway/index.js                  |  3 +-
 test/http-api/block.js                 |  3 +-
 test/http-api/interface.js             |  6 ++--
 test/http-api/routes.js                |  3 +-
 test/utils/interface-common-factory.js |  3 +-
 29 files changed, 91 insertions(+), 45 deletions(-)

diff --git a/package.json b/package.json
index 0cb519b6c6..87af08abe5 100644
--- a/package.json
+++ b/package.json
@@ -60,6 +60,7 @@
     "@hapi/ammo": "^3.1.0",
     "@hapi/hapi": "^18.3.1",
     "@hapi/joi": "^15.0.1",
+    "array-shuffle": "^1.0.1",
     "async": "^2.6.1",
     "async-iterator-all": "^1.0.0",
     "async-iterator-to-pull-stream": "^1.1.0",
diff --git a/src/core/preload.js b/src/core/preload.js
index 97081fd83c..983b0445d1 100644
--- a/src/core/preload.js
+++ b/src/core/preload.js
@@ -5,6 +5,7 @@ const retry = require('async/retry')
 const toUri = require('multiaddr-to-uri')
 const debug = require('debug')
 const CID = require('cids')
+const shuffle = require('array-shuffle')
 const preload = require('./runtime/preload-nodejs')
 
 const log = debug('ipfs:preload')
@@ -44,7 +45,7 @@ module.exports = self => {
       }
     }
 
-    const fallbackApiUris = Array.from(apiUris)
+    const fallbackApiUris = shuffle(apiUris)
     let request
     const now = Date.now()
 
diff --git a/test/core/bitswap.spec.js b/test/core/bitswap.spec.js
index e5bce37387..9809393bb0 100644
--- a/test/core/bitswap.spec.js
+++ b/test/core/bitswap.spec.js
@@ -76,7 +76,8 @@ function addNode (fDaemon, inProcNode, callback) {
         }
       },
       Bootstrap: []
-    }
+    },
+    preload: { enabled: false }
   }, (err, ipfsd) => {
     expect(err).to.not.exist()
     nodes.push(ipfsd)
diff --git a/test/core/block.spec.js b/test/core/block.spec.js
index 7c052a7802..64b6c7f385 100644
--- a/test/core/block.spec.js
+++ b/test/core/block.spec.js
@@ -22,7 +22,8 @@ describe('block', () => {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/bootstrap.spec.js b/test/core/bootstrap.spec.js
index 72cae70840..57bd73bad2 100644
--- a/test/core/bootstrap.spec.js
+++ b/test/core/bootstrap.spec.js
@@ -29,7 +29,8 @@ describe('bootstrap', () => {
         Addresses: {
           Swarm: ['/ip4/127.0.0.1/tcp/0']
         }
-      }
+      },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/circuit-relay.js b/test/core/circuit-relay.js
index 1566ce04b1..a649f258c9 100644
--- a/test/core/circuit-relay.js
+++ b/test/core/circuit-relay.js
@@ -52,7 +52,8 @@ function setupInProcNode (addrs, hop, callback) {
       Addresses: {
         Swarm: addrs
       }
-    })
+    }),
+    preload: { enabled: false }
   }, (err, ipfsd) => {
     expect(err).to.not.exist()
     ipfsd.api.id((err, id) => {
diff --git a/test/core/create-node.spec.js b/test/core/create-node.spec.js
index 5d41eb980d..0595def717 100644
--- a/test/core/create-node.spec.js
+++ b/test/core/create-node.spec.js
@@ -38,7 +38,8 @@ describe('create node', function () {
         Addresses: {
           Swarm: []
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('start', (err) => {
@@ -63,7 +64,8 @@ describe('create node', function () {
         Addresses: {
           Swarm: []
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('start', (err) => {
@@ -117,7 +119,8 @@ describe('create node', function () {
         Addresses: {
           Swarm: []
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('start', (err) => {
@@ -139,7 +142,8 @@ describe('create node', function () {
 
     const ipfs = new IPFS({
       silent: true,
-      repo: tempRepo
+      repo: tempRepo,
+      preload: { enabled: false }
     })
 
     ipfs.on('ready', () => {
@@ -161,7 +165,8 @@ describe('create node', function () {
         Addresses: {
           Swarm: []
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     const shouldHappenOnce = () => {
@@ -194,7 +199,8 @@ describe('create node', function () {
         Addresses: {
           Swarm: []
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     let happened = false
@@ -225,7 +231,8 @@ describe('create node', function () {
           Swarm: []
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('error', done)
@@ -247,7 +254,8 @@ describe('create node', function () {
           Swarm: []
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('error', done)
@@ -266,7 +274,8 @@ describe('create node', function () {
           Swarm: ['/ip4/127.0.0.1/tcp/9977']
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('start', (err) => {
@@ -292,7 +301,8 @@ describe('create node', function () {
           Swarm: []
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
 
     series([
@@ -313,7 +323,8 @@ describe('create node', function () {
           Swarm: []
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
 
     node.once('ready', () => {
@@ -333,7 +344,8 @@ describe('create node', function () {
           Swarm: []
         },
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     }
 
     let node = new IPFS(options)
@@ -368,7 +380,8 @@ describe('create node', function () {
             ]
           },
           Bootstrap: []
-        }
+        },
+        preload: { enabled: false }
       })
     }
 
@@ -417,7 +430,8 @@ describe('create node', function () {
 
     const node = new IPFS({
       repo: tempRepo,
-      ipld: {}
+      ipld: {},
+      preload: { enabled: false }
     })
 
     node.once('start', (err) => {
diff --git a/test/core/dag.spec.js b/test/core/dag.spec.js
index 4b9cf089a2..55c4bad299 100644
--- a/test/core/dag.spec.js
+++ b/test/core/dag.spec.js
@@ -20,7 +20,8 @@ describe('dag', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/dht.spec.js b/test/core/dht.spec.js
index ba437a28bc..2eedd6205e 100644
--- a/test/core/dht.spec.js
+++ b/test/core/dht.spec.js
@@ -27,7 +27,8 @@ describe.skip('dht', () => {
         initOptions: { bits: 512 },
         config: {
           Bootstrap: []
-        }
+        },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         ipfsd = _ipfsd
diff --git a/test/core/files-sharding.spec.js b/test/core/files-sharding.spec.js
index 9289d78dff..c34213b465 100644
--- a/test/core/files-sharding.spec.js
+++ b/test/core/files-sharding.spec.js
@@ -47,7 +47,8 @@ describe('files directory (sharding tests)', () => {
               Enabled: false
             }
           }
-        }
+        },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         ipfsd = _ipfsd
@@ -99,7 +100,8 @@ describe('files directory (sharding tests)', () => {
               Enabled: false
             }
           }
-        }
+        },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         ipfsd = _ipfsd
diff --git a/test/core/files.spec.js b/test/core/files.spec.js
index 0558e88419..6d7e496706 100644
--- a/test/core/files.spec.js
+++ b/test/core/files.spec.js
@@ -21,7 +21,8 @@ describe('files', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/init.spec.js b/test/core/init.spec.js
index 797cbced03..706de3d6b6 100644
--- a/test/core/init.spec.js
+++ b/test/core/init.spec.js
@@ -27,7 +27,8 @@ describe('init', () => {
     ipfs = new IPFS({
       repo: repo,
       init: false,
-      start: false
+      start: false,
+      preload: { enabled: false }
     })
   })
 
diff --git a/test/core/kad-dht.node.js b/test/core/kad-dht.node.js
index d06392abbc..b7cc90896e 100644
--- a/test/core/kad-dht.node.js
+++ b/test/core/kad-dht.node.js
@@ -28,7 +28,8 @@ function createNode (callback) {
   f.spawn({
     exec: path.resolve(`${__dirname}/../../src/cli/bin.js`),
     config,
-    initOptions: { bits: 512 }
+    initOptions: { bits: 512 },
+    args: ['--preload-enabled=false']
   }, callback)
 }
 
diff --git a/test/core/key-exchange.js b/test/core/key-exchange.js
index 1d9bde7ee8..40e17efc79 100644
--- a/test/core/key-exchange.js
+++ b/test/core/key-exchange.js
@@ -23,7 +23,8 @@ describe('key exchange', () => {
     repo = createTempRepo()
     ipfs = new IPFS({
       repo: repo,
-      pass: hat()
+      pass: hat(),
+      preload: { enabled: false }
     })
     ipfs.on('ready', () => done())
   })
diff --git a/test/core/name-pubsub.js b/test/core/name-pubsub.js
index 884d0a55b7..0af2a59eea 100644
--- a/test/core/name-pubsub.js
+++ b/test/core/name-pubsub.js
@@ -51,7 +51,8 @@ describe('name-pubsub', function () {
             Enabled: false
           }
         }
-      }
+      },
+      preload: { enabled: false }
     }, callback)
   }
 
diff --git a/test/core/name.spec.js b/test/core/name.spec.js
index ec2e3f4706..93966cab8e 100644
--- a/test/core/name.spec.js
+++ b/test/core/name.spec.js
@@ -47,7 +47,8 @@ describe('name', function () {
       df.spawn({
         exec: IPFS,
         args: [`--pass ${hat()}`, '--offline'],
-        config: { Bootstrap: [] }
+        config: { Bootstrap: [] },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         ipfsd = _ipfsd
@@ -195,7 +196,8 @@ describe('name', function () {
               Enabled: false
             }
           }
-        }
+        },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         ipfsd = _ipfsd
@@ -371,7 +373,8 @@ describe('name', function () {
               Enabled: false
             }
           }
-        }
+        },
+        preload: { enabled: false }
       }, (err, _ipfsd) => {
         expect(err).to.not.exist()
         node = _ipfsd.api
diff --git a/test/core/object.spec.js b/test/core/object.spec.js
index 2cbbbd2ada..0d931c6961 100644
--- a/test/core/object.spec.js
+++ b/test/core/object.spec.js
@@ -22,7 +22,8 @@ describe('object', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/pin-set.js b/test/core/pin-set.js
index 088d248a12..3df518bc27 100644
--- a/test/core/pin-set.js
+++ b/test/core/pin-set.js
@@ -82,7 +82,8 @@ describe('pinSet', function () {
             Enabled: false
           }
         }
-      }
+      },
+      preload: { enabled: false }
     })
     ipfs.on('ready', () => {
       pinSet = createPinSet(ipfs.dag)
diff --git a/test/core/pin.js b/test/core/pin.js
index b6ad6a220c..c41fe2f715 100644
--- a/test/core/pin.js
+++ b/test/core/pin.js
@@ -75,7 +75,8 @@ describe('pin', function () {
       repo,
       config: {
         Bootstrap: []
-      }
+      },
+      preload: { enabled: false }
     })
     ipfs.on('ready', () => {
       pin = ipfs.pin
diff --git a/test/core/pin.spec.js b/test/core/pin.spec.js
index bd2126c7bb..c03b62ef29 100644
--- a/test/core/pin.spec.js
+++ b/test/core/pin.spec.js
@@ -20,7 +20,8 @@ describe('pin', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/ping.spec.js b/test/core/ping.spec.js
index 6d04792013..009d7ec595 100644
--- a/test/core/ping.spec.js
+++ b/test/core/ping.spec.js
@@ -34,7 +34,8 @@ function spawnNode ({ dht = false, type = 'js' }, cb) {
   factory.spawn({
     args,
     config,
-    initOptions: { bits: 512 }
+    initOptions: { bits: 512 },
+    preload: { enabled: false }
   }, cb)
 }
 
diff --git a/test/core/stats.spec.js b/test/core/stats.spec.js
index a89f9174c5..df29295678 100644
--- a/test/core/stats.spec.js
+++ b/test/core/stats.spec.js
@@ -21,7 +21,8 @@ describe('stats', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/swarm.spec.js b/test/core/swarm.spec.js
index fb35cfff33..b6a774a2a2 100644
--- a/test/core/swarm.spec.js
+++ b/test/core/swarm.spec.js
@@ -20,7 +20,8 @@ describe('swarm', function () {
     factory.spawn({
       exec: IPFS,
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      preload: { enabled: false }
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/core/utils.js b/test/core/utils.js
index c05c53a916..4da679f18a 100644
--- a/test/core/utils.js
+++ b/test/core/utils.js
@@ -123,7 +123,8 @@ describe('utils', () => {
         repo,
         config: {
           Bootstrap: []
-        }
+        },
+        preload: { enabled: false }
       })
       node.once('ready', () => node.add(fixtures, done))
     })
diff --git a/test/gateway/index.js b/test/gateway/index.js
index 3c13a7452d..22fc6ea7b6 100644
--- a/test/gateway/index.js
+++ b/test/gateway/index.js
@@ -49,7 +49,8 @@ describe('HTTP Gateway', function () {
             Enabled: false
           }
         }
-      }
+      },
+      preload: { enabled: false }
     })
 
     const content = (name) => ({
diff --git a/test/http-api/block.js b/test/http-api/block.js
index 1efce99d05..a0e2c5363a 100644
--- a/test/http-api/block.js
+++ b/test/http-api/block.js
@@ -21,7 +21,8 @@ describe('block endpoint', () => {
 
     df.spawn({
       initOptions: { bits: 512 },
-      config: { Bootstrap: [] }
+      config: { Bootstrap: [] },
+      args: []
     }, (err, _ipfsd) => {
       expect(err).to.not.exist()
       ipfsd = _ipfsd
diff --git a/test/http-api/interface.js b/test/http-api/interface.js
index d1607dfb71..cedeccee06 100644
--- a/test/http-api/interface.js
+++ b/test/http-api/interface.js
@@ -46,7 +46,8 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => {
             Enabled: false
           }
         }
-      }
+      },
+      preload: { enabled: false }
     }
   }), {
     skip: {
@@ -72,7 +73,8 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => {
             Enabled: false
           }
         }
-      }
+      },
+      preload: { enabled: false }
     }
   }))
 
diff --git a/test/http-api/routes.js b/test/http-api/routes.js
index d79572dc18..369930381f 100644
--- a/test/http-api/routes.js
+++ b/test/http-api/routes.js
@@ -28,7 +28,8 @@ describe('HTTP API', () => {
       config,
       EXPERIMENTAL: {
         pubsub: true
-      }
+      },
+      preload: { enabled: false }
     })
     await ncp(repoExample, repoTests)
     await http.api.start()
diff --git a/test/utils/interface-common-factory.js b/test/utils/interface-common-factory.js
index 756e79c1cb..8df86419a7 100644
--- a/test/utils/interface-common-factory.js
+++ b/test/utils/interface-common-factory.js
@@ -22,7 +22,8 @@ function createFactory (options) {
           Enabled: false
         }
       }
-    }
+    },
+    preload: { enabled: false }
   }
 
   if (options.factoryOptions.type !== 'proc') {