perf: lazy load IPLD formats (#1704)

This PR uses the new `loadFormat` option for IPLD to lazily require IPLD formats in order to reduce the startup time for the node.

If you're feeling like you've seen this before then, for reference:

The PR ipld/js-ipld#164 undid the work done in ipld/js-ipld#145 and ipld/js-ipld#178 re-enabled the ability to do so. This PR makes use of this new ability to lazy load the formats.

License: MIT
Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>

package.json

@@ -108,12 +108,11 @@
     "ipfs-repo": "~0.25.0",
     "ipfs-unixfs": "~0.1.16",
     "ipfs-unixfs-engine": "~0.33.0",
-    "ipld": "~0.19.1",
+    "ipld": "~0.19.3",
     "ipld-bitcoin": "~0.1.8",
     "ipld-dag-pb": "~0.14.11",
     "ipld-ethereum": "^2.0.1",
     "ipld-git": "~0.2.2",
-    "ipld-raw": "^2.0.1",
     "ipld-zcash": "~0.1.6",
     "ipns": "~0.3.0",
     "is-ipfs": "~0.4.7",

src/core/index.js

@@ -15,21 +15,6 @@ const debug = require('debug')
 const extend = require('deep-extend')
 const EventEmitter = require('events')
 
-// All known IPLD formats
-const ipldBitcoin = require('ipld-bitcoin')
-const ipldDagCbor = require('ipld-dag-cbor')
-const ipldDagPb = require('ipld-dag-pb')
-const ipldEthAccountSnapshot = require('ipld-ethereum').ethAccountSnapshot
-const ipldEthBlock = require('ipld-ethereum').ethBlock
-const ipldEthBlockList = require('ipld-ethereum').ethBlockList
-const ipldEthStateTrie = require('ipld-ethereum').ethStateTrie
-const ipldEthStorageTrie = require('ipld-ethereum').ethStorageTrie
-const ipldEthTrie = require('ipld-ethereum').ethTxTrie
-const ipldEthTx = require('ipld-ethereum').ethTx
-const ipldGit = require('ipld-git')
-const ipldRaw = require('ipld-raw')
-const ipldZcash = require('ipld-zcash')
-
 const config = require('./config')
 const boot = require('./boot')
 const components = require('./components')
@@ -39,6 +24,40 @@ const defaultRepo = require('./runtime/repo-nodejs')
 const preload = require('./preload')
 const mfsPreload = require('./mfs-preload')
 
+// All known (non-default) IPLD formats
+const IpldFormats = {
+  get 'bitcoin-block' () {
+    return require('ipld-bitcoin')
+  },
+  get 'eth-account-snapshot' () {
+    return require('ipld-ethereum').ethAccountSnapshot
+  },
+  get 'eth-block' () {
+    return require('ipld-ethereum').ethBlock
+  },
+  get 'eth-block-list' () {
+    return require('ipld-ethereum').ethBlockList
+  },
+  get 'eth-state-trie' () {
+    return require('ipld-ethereum').ethStateTrie
+  },
+  get 'eth-storage-trie' () {
+    return require('ipld-ethereum').ethStorageTrie
+  },
+  get 'eth-tx' () {
+    return require('ipld-ethereum').ethTx
+  },
+  get 'eth-tx-trie' () {
+    return require('ipld-ethereum').ethTxTrie
+  },
+  get 'git-raw' () {
+    return require('ipld-git')
+  },
+  get 'zcash-block' () {
+    return require('ipld-zcash')
+  }
+}
+
 class IPFS extends EventEmitter {
   constructor (options) {
     super()
@@ -99,11 +118,11 @@ class IPFS extends EventEmitter {
     this._blockService = new BlockService(this._repo)
     this._ipld = new Ipld({
       blockService: this._blockService,
-      formats: [
-        ipldBitcoin, ipldDagCbor, ipldDagPb, ipldEthAccountSnapshot,
-        ipldEthBlock, ipldEthBlockList, ipldEthStateTrie, ipldEthStorageTrie,
-        ipldEthTrie, ipldEthTx, ipldGit, ipldRaw, ipldZcash
-      ]
+      loadFormat: (codec, callback) => {
+        this.log('Loading IPLD format', codec)
+        if (IpldFormats[codec]) return callback(null, IpldFormats[codec])
+        callback(new Error(`Missing IPLD format "${codec}"`))
+      }
     })
     this._preload = preload(this)
     this._mfsPreload = mfsPreload(this)