Skip to content

Commit

Permalink
feat: randomly pick preload node (ipfs#2194)
Browse files Browse the repository at this point in the history 
This PR updates the preloader to randomly pick a node when preloading data.

It actually shuffles the list of _available_ preload nodes for every preload
request, and works its way through. This helps spread the load between available
preload nodes.

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>
  • Loading branch information
@alanshaw
alanshaw committed Jun 28, 2019
1 parent 1aaaab9 commit f596b01
Show file tree
Hide file tree
Showing 29 changed files with 91 additions and 45 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
"@hapi/ammo": "^3.1.0",
"@hapi/hapi": "^18.3.1",
"@hapi/joi": "^15.0.1",
"array-shuffle": "^1.0.1",
"async": "^2.6.1",
"async-iterator-all": "^1.0.0",
"async-iterator-to-pull-stream": "^1.1.0",
Expand Down
3 changes: 2 additions & 1 deletion src/core/preload.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const retry = require('async/retry')
const toUri = require('multiaddr-to-uri')
const debug = require('debug')
const CID = require('cids')
const shuffle = require('array-shuffle')
const preload = require('./runtime/preload-nodejs')

const log = debug('ipfs:preload')
Expand Down Expand Up @@ -44,7 +45,7 @@ module.exports = self => {
}
}

const fallbackApiUris = Array.from(apiUris)
const fallbackApiUris = shuffle(apiUris)
let request
const now = Date.now()

Expand Down
3 changes: 2 additions & 1 deletion test/core/bitswap.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ function addNode (fDaemon, inProcNode, callback) {
}
},
Bootstrap: []
}
},
preload: { enabled: false }
}, (err, ipfsd) => {
expect(err).to.not.exist()
nodes.push(ipfsd)
Expand Down
3 changes: 2 additions & 1 deletion test/core/block.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ describe('block', () => {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/bootstrap.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ describe('bootstrap', () => {
Addresses: {
Swarm: ['/ip4/127.0.0.1/tcp/0']
}
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/circuit-relay.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ function setupInProcNode (addrs, hop, callback) {
Addresses: {
Swarm: addrs
}
})
}),
preload: { enabled: false }
}, (err, ipfsd) => {
expect(err).to.not.exist()
ipfsd.api.id((err, id) => {
Expand Down
42 changes: 28 additions & 14 deletions test/core/create-node.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ describe('create node', function () {
Addresses: {
Swarm: []
}
}
},
preload: { enabled: false }
})

node.once('start', (err) => {
Expand All @@ -63,7 +64,8 @@ describe('create node', function () {
Addresses: {
Swarm: []
}
}
},
preload: { enabled: false }
})

node.once('start', (err) => {
Expand Down Expand Up @@ -117,7 +119,8 @@ describe('create node', function () {
Addresses: {
Swarm: []
}
}
},
preload: { enabled: false }
})

node.once('start', (err) => {
Expand All @@ -139,7 +142,8 @@ describe('create node', function () {

const ipfs = new IPFS({
silent: true,
repo: tempRepo
repo: tempRepo,
preload: { enabled: false }
})

ipfs.on('ready', () => {
Expand All @@ -161,7 +165,8 @@ describe('create node', function () {
Addresses: {
Swarm: []
}
}
},
preload: { enabled: false }
})

const shouldHappenOnce = () => {
Expand Down Expand Up @@ -194,7 +199,8 @@ describe('create node', function () {
Addresses: {
Swarm: []
}
}
},
preload: { enabled: false }
})

let happened = false
Expand Down Expand Up @@ -225,7 +231,8 @@ describe('create node', function () {
Swarm: []
},
Bootstrap: []
}
},
preload: { enabled: false }
})

node.once('error', done)
Expand All @@ -247,7 +254,8 @@ describe('create node', function () {
Swarm: []
},
Bootstrap: []
}
},
preload: { enabled: false }
})

node.once('error', done)
Expand All @@ -266,7 +274,8 @@ describe('create node', function () {
Swarm: ['/ip4/127.0.0.1/tcp/9977']
},
Bootstrap: []
}
},
preload: { enabled: false }
})

node.once('start', (err) => {
Expand All @@ -292,7 +301,8 @@ describe('create node', function () {
Swarm: []
},
Bootstrap: []
}
},
preload: { enabled: false }
})

series([
Expand All @@ -313,7 +323,8 @@ describe('create node', function () {
Swarm: []
},
Bootstrap: []
}
},
preload: { enabled: false }
})

node.once('ready', () => {
Expand All @@ -333,7 +344,8 @@ describe('create node', function () {
Swarm: []
},
Bootstrap: []
}
},
preload: { enabled: false }
}

let node = new IPFS(options)
Expand Down Expand Up @@ -368,7 +380,8 @@ describe('create node', function () {
]
},
Bootstrap: []
}
},
preload: { enabled: false }
})
}

Expand Down Expand Up @@ -417,7 +430,8 @@ describe('create node', function () {

const node = new IPFS({
repo: tempRepo,
ipld: {}
ipld: {},
preload: { enabled: false }
})

node.once('start', (err) => {
Expand Down
3 changes: 2 additions & 1 deletion test/core/dag.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ describe('dag', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/dht.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ describe.skip('dht', () => {
initOptions: { bits: 512 },
config: {
Bootstrap: []
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
6 changes: 4 additions & 2 deletions test/core/files-sharding.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ describe('files directory (sharding tests)', () => {
Enabled: false
}
}
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down Expand Up @@ -99,7 +100,8 @@ describe('files directory (sharding tests)', () => {
Enabled: false
}
}
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/files.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ describe('files', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/init.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ describe('init', () => {
ipfs = new IPFS({
repo: repo,
init: false,
start: false
start: false,
preload: { enabled: false }
})
})

Expand Down
3 changes: 2 additions & 1 deletion test/core/kad-dht.node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ function createNode (callback) {
f.spawn({
exec: path.resolve(`${__dirname}/../../src/cli/bin.js`),
config,
initOptions: { bits: 512 }
initOptions: { bits: 512 },
args: ['--preload-enabled=false']
}, callback)
}

Expand Down
3 changes: 2 additions & 1 deletion test/core/key-exchange.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ describe('key exchange', () => {
repo = createTempRepo()
ipfs = new IPFS({
repo: repo,
pass: hat()
pass: hat(),
preload: { enabled: false }
})
ipfs.on('ready', () => done())
})
Expand Down
3 changes: 2 additions & 1 deletion test/core/name-pubsub.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ describe('name-pubsub', function () {
Enabled: false
}
}
}
},
preload: { enabled: false }
}, callback)
}

Expand Down
9 changes: 6 additions & 3 deletions test/core/name.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ describe('name', function () {
df.spawn({
exec: IPFS,
args: [`--pass ${hat()}`, '--offline'],
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down Expand Up @@ -195,7 +196,8 @@ describe('name', function () {
Enabled: false
}
}
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down Expand Up @@ -371,7 +373,8 @@ describe('name', function () {
Enabled: false
}
}
}
},
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
node = _ipfsd.api
Expand Down
3 changes: 2 additions & 1 deletion test/core/object.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ describe('object', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/pin-set.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ describe('pinSet', function () {
Enabled: false
}
}
}
},
preload: { enabled: false }
})
ipfs.on('ready', () => {
pinSet = createPinSet(ipfs.dag)
Expand Down
3 changes: 2 additions & 1 deletion test/core/pin.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ describe('pin', function () {
repo,
config: {
Bootstrap: []
}
},
preload: { enabled: false }
})
ipfs.on('ready', () => {
pin = ipfs.pin
Expand Down
3 changes: 2 additions & 1 deletion test/core/pin.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ describe('pin', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/ping.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ function spawnNode ({ dht = false, type = 'js' }, cb) {
factory.spawn({
args,
config,
initOptions: { bits: 512 }
initOptions: { bits: 512 },
preload: { enabled: false }
}, cb)
}

Expand Down
3 changes: 2 additions & 1 deletion test/core/stats.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ describe('stats', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
3 changes: 2 additions & 1 deletion test/core/swarm.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ describe('swarm', function () {
factory.spawn({
exec: IPFS,
initOptions: { bits: 512 },
config: { Bootstrap: [] }
config: { Bootstrap: [] },
preload: { enabled: false }
}, (err, _ipfsd) => {
expect(err).to.not.exist()
ipfsd = _ipfsd
Expand Down
Loading

0 comments on commit f596b01

Please sign in to comment.