feat: preload on content fetch requests

package.json

@@ -58,7 +58,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^15.0.0",
+    "aegir": "^15.1.0",
     "buffer-loader": "~0.0.1",
     "chai": "^4.1.2",
     "delay": "^3.0.0",
@@ -70,8 +70,8 @@
     "expose-loader": "~0.7.5",
     "form-data": "^2.3.2",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.72.0",
-    "ipfsd-ctl": "~0.37.5",
+    "interface-ipfs-core": "~0.72.1",
+    "ipfsd-ctl": "~0.38.0",
     "mocha": "^5.2.0",
     "ncp": "^2.0.0",
     "nexpect": "~0.5.0",
@@ -93,7 +93,7 @@
     "byteman": "^1.3.5",
     "cids": "~0.5.3",
     "debug": "^3.1.0",
-    "file-type": "^8.0.0",
+    "file-type": "^8.1.0",
     "filesize": "^3.6.1",
     "fnv1a": "^1.0.1",
     "fsm-event": "^2.1.0",
@@ -104,30 +104,30 @@
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.4.2",
-    "ipfs-api": "^22.2.1",
-    "ipfs-bitswap": "~0.20.2",
+    "ipfs-api": "^22.2.4",
+    "ipfs-bitswap": "~0.20.3",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
-    "ipfs-mfs": "~0.2.2",
+    "ipfs-mfs": "~0.2.3",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
     "ipfs-unixfs-engine": "~0.31.3",
     "ipld": "~0.17.3",
     "ipld-dag-cbor": "~0.12.1",
-    "ipld-dag-pb": "~0.14.5",
-    "is-ipfs": "~0.3.2",
+    "ipld-dag-pb": "~0.14.6",
+    "is-ipfs": "~0.4.2",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^2.0.0",
-    "libp2p": "~0.22.0",
+    "libp2p": "~0.23.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-circuit": "~0.2.0",
     "libp2p-floodsub": "~0.15.0",
-    "libp2p-kad-dht": "~0.10.0",
+    "libp2p-kad-dht": "~0.10.1",
     "libp2p-keychain": "~0.3.1",
     "libp2p-mdns": "~0.12.0",
     "libp2p-mplex": "~0.8.0",
@@ -138,7 +138,7 @@
     "libp2p-websockets": "~0.12.0",
     "lodash": "^4.17.10",
     "mafmt": "^6.0.0",
-    "mime-types": "^2.1.18",
+    "mime-types": "^2.1.19",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
     "multiaddr-to-uri": "^4.0.0",

src/core/components/block.js

@@ -9,8 +9,20 @@ const promisify = require('promisify-es6')
 
 module.exports = function block (self) {
   return {
-    get: promisify((cid, callback) => {
+    get: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
       cid = cleanCid(cid)
+
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, callback)
     }),
     put: promisify((block, options, callback) => {
@@ -65,9 +77,18 @@ module.exports = function block (self) {
       cid = cleanCid(cid)
       self._blockService.delete(cid, callback)
     }),
-    stat: promisify((cid, callback) => {
+    stat: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
       cid = cleanCid(cid)
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, (err, block) => {
         if (err) {
           return callback(err)

src/core/components/dag.js

@@ -66,6 +66,10 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, path, options, callback)
     }),
 
@@ -100,17 +104,28 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       pull(
         self._ipld.treeStream(cid, path, options),
         pull.collect(callback)
       )
     }),
 
     // TODO - use IPLD selectors once they are implemented
-    _getRecursive: promisify((multihash, callback) => {
+    _getRecursive: promisify((multihash, options, callback) => {
       // gets flat array of all DAGNodes in tree given by multihash
 
-      self.dag.get(new CID(multihash), (err, res) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
+      self.dag.get(new CID(multihash), '', options, (err, res) => {
         if (err) { return callback(err) }
 
         mapAsync(res.value.links, (link, cb) => {

src/core/components/files.js

@@ -182,11 +182,17 @@ module.exports = function files (self) {
       throw new Error('You must supply an ipfsPath')
     }
 
+    options = options || {}
+
     ipfsPath = normalizePath(ipfsPath)
     const pathComponents = ipfsPath.split('/')
     const restPath = normalizePath(pathComponents.slice(1).join('/'))
     const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath)
 
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
+
     const d = deferred.source()
 
     pull(
@@ -213,16 +219,21 @@ module.exports = function files (self) {
   }
 
   function _lsPullStreamImmutable (ipfsPath, options) {
+    options = options || {}
+
     const path = normalizePath(ipfsPath)
-    const recursive = options && options.recursive
-    const pathDepth = path.split('/').length
+    const recursive = options.recursive
+    const pathComponents = path.split('/')
+    const pathDepth = pathComponents.length
     const maxDepth = recursive ? global.Infinity : pathDepth
-    const opts = Object.assign({}, {
-      maxDepth: maxDepth
-    }, options)
+    options.maxDepth = options.maxDepth || maxDepth
+
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
 
     return pull(
-      exporter(ipfsPath, self._ipld, opts),
+      exporter(ipfsPath, self._ipld, options),
       pull.filter(node =>
         recursive ? node.depth >= pathDepth : node.depth === pathDepth
       ),
@@ -334,8 +345,11 @@ module.exports = function files (self) {
         options = {}
       }
 
-      if (typeof callback !== 'function') {
-        throw new Error('Please supply a callback to ipfs.files.get')
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
       }
 
       pull(
@@ -359,6 +373,13 @@ module.exports = function files (self) {
     }),
 
     getReadableStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(
         pull(
           exporter(ipfsPath, self._ipld, options),
@@ -375,6 +396,13 @@ module.exports = function files (self) {
     },
 
     getPullStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return exporter(ipfsPath, self._ipld, options)
     },
 
@@ -384,6 +412,13 @@ module.exports = function files (self) {
         options = {}
       }
 
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       pull(
         _lsPullStreamImmutable(ipfsPath, options),
         pull.collect((err, values) => {
@@ -397,6 +432,13 @@ module.exports = function files (self) {
     }),
 
     lsReadableStreamImmutable: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
     },
 

src/core/components/mfs.js

@@ -7,7 +7,23 @@ module.exports = self => {
   const mfsSelf = Object.assign({}, self)
 
   // A patched dag API to ensure preload doesn't happen for MFS operations
+  // (MFS is preloaded periodically)
   mfsSelf.dag = Object.assign({}, self.dag, {
+    get: promisify((cid, path, opts, cb) => {
+      if (typeof path === 'function') {
+        cb = path
+        path = undefined
+      }
+
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      opts = Object.assign({}, opts, { preload: false })
+
+      return self.dag.get(cid, path, opts, cb)
+    }),
     put: promisify((node, opts, cb) => {
       if (typeof opts === 'function') {
         cb = opts

src/core/components/object.js

@@ -1,14 +1,14 @@
 'use strict'
 
 const waterfall = require('async/waterfall')
+const setImmediate = require('async/setImmediate')
 const promisify = require('promisify-es6')
 const dagPB = require('ipld-dag-pb')
 const DAGNode = dagPB.DAGNode
 const DAGLink = dagPB.DAGLink
 const CID = require('cids')
 const mh = require('multihashes')
 const Unixfs = require('ipfs-unixfs')
-const assert = require('assert')
 
 function normalizeMultihash (multihash, enc) {
   if (typeof multihash === 'string') {
@@ -116,7 +116,9 @@ module.exports = function object (self) {
       let data
 
       if (template) {
-        assert(template === 'unixfs-dir', 'unkown template')
+        if (template !== 'unixfs-dir') {
+          return setImmediate(() => callback(new Error('unknown template')))
+        }
         data = (new Unixfs('directory')).marshal()
       } else {
         data = Buffer.alloc(0)
@@ -221,6 +223,10 @@ module.exports = function object (self) {
         cid = cid.toV1()
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, (err, result) => {
         if (err) {
           return callback(err)

src/core/components/pin-set.js

@@ -74,7 +74,7 @@ exports = module.exports = function (dag) {
 
           seen[bs58Link] = true
 
-          dag.get(multihash, (err, res) => {
+          dag.get(multihash, '', { preload: false }, (err, res) => {
             if (err) { return someCb(err) }
             searchChildren(res.value, someCb)
           })
@@ -184,7 +184,7 @@ exports = module.exports = function (dag) {
         return callback(new Error('No link found with name ' + name))
       }
 
-      dag.get(link.multihash, (err, res) => {
+      dag.get(link.multihash, '', { preload: false }, (err, res) => {
         if (err) { return callback(err) }
         const keys = []
         const step = link => keys.push(link.multihash)
@@ -211,7 +211,7 @@ exports = module.exports = function (dag) {
 
           if (!emptyKey.equals(linkHash)) {
             // walk the links of this fanout bin
-            return dag.get(linkHash, (err, res) => {
+            return dag.get(linkHash, '', { preload: false }, (err, res) => {
               if (err) { return eachCb(err) }
               pinSet.walkItems(res.value, step, eachCb)
             })

src/core/components/pin.js

@@ -360,7 +360,7 @@ module.exports = (self) => {
         (_, cb) => repo.datastore.has(pinDataStoreKey, cb),
         (has, cb) => has ? cb() : cb(new Error('No pins to load')),
         (cb) => repo.datastore.get(pinDataStoreKey, cb),
-        (mh, cb) => dag.get(new CID(mh), cb)
+        (mh, cb) => dag.get(new CID(mh), '', { preload: false }, cb)
       ], (err, pinRoot) => {
         if (err) {
           if (err.message === 'No pins to load') {

test/cli/bootstrap.js

@@ -30,8 +30,6 @@ describe('bootstrap', () => runOnAndOff((thing) => {
     '/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
     '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
     '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
-    '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
     '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
     '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
   ]
@@ -54,8 +52,6 @@ describe('bootstrap', () => runOnAndOff((thing) => {
     '/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
     '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
     '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
-    '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
     '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
     '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
     '/ip4/111.111.111.111/tcp/1001/ipfs/QmcyFFKfLDGJKwufn2GeitxvhricsBQyNKTkrD14psikoD'