refactor: async iterables

package.json

@@ -15,8 +15,7 @@
   ],
   "main": "src/core/index.js",
   "browser": {
-    "./src/core/components/init-assets.js": false,
-    "./src/core/runtime/add-from-fs-nodejs.js": "./src/core/runtime/add-from-fs-browser.js",
+    "./src/core/runtime/init-assets-nodejs.js": "./src/core/runtime/init-assets-browser.js",
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
@@ -25,7 +24,8 @@
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
     "./src/core/runtime/ipld-nodejs.js": "./src/core/runtime/ipld-browser.js",
     "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
-    "stream": "readable-stream"
+    "stream": "readable-stream",
+    "ipfs-utils/src/files/glob-source": false
   },
   "browser-all-ipld-formats": {
     "./src/core/runtime/ipld-browser.js": "./src/core/runtime/ipld-browser-all.js"
@@ -95,20 +95,19 @@
     "glob": "^7.1.3",
     "hapi-pino": "^6.1.0",
     "hashlru": "^2.3.0",
-    "human-to-milliseconds": "^2.0.0",
     "interface-datastore": "~0.8.0",
     "ipfs-bitswap": "^0.26.0",
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.16.0",
-    "ipfs-http-client": "^40.0.1",
+    "ipfs-http-client": "github:ipfs/js-ipfs-http-client#refactor/async-iterables2",
     "ipfs-http-response": "~0.4.0",
-    "ipfs-mfs": "^0.13.2",
+    "ipfs-mfs": "github:ipfs/js-ipfs-mfs#refactor/remove-streams",
     "ipfs-multipart": "^0.2.0",
     "ipfs-repo": "^0.30.0",
     "ipfs-unixfs": "~0.1.16",
     "ipfs-unixfs-exporter": "^0.38.0",
     "ipfs-unixfs-importer": "^0.40.0",
-    "ipfs-utils": "~0.4.0",
+    "ipfs-utils": "^0.5.0",
     "ipld": "~0.25.0",
     "ipld-bitcoin": "~0.3.0",
     "ipld-dag-cbor": "~0.15.0",
@@ -125,34 +124,35 @@
     "iso-url": "~0.4.6",
     "it-pipe": "^1.0.1",
     "it-to-stream": "^0.1.1",
+    "iterable-ndjson": "^1.1.0",
     "jsondiffpatch": "~0.3.11",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
     "ky": "^0.15.0",
     "ky-universal": "~0.3.0",
-    "libp2p": "^0.26.2",
-    "libp2p-bootstrap": "~0.9.3",
-    "libp2p-crypto": "^0.16.2",
+    "libp2p": "^0.27.0-pre.1",
+    "libp2p-bootstrap": "^0.10.2",
+    "libp2p-crypto": "^0.17.1",
     "libp2p-delegated-content-routing": "^0.4.1",
-    "libp2p-delegated-peer-routing": "^0.3.1",
-    "libp2p-floodsub": "^0.18.0",
-    "libp2p-gossipsub": "~0.0.5",
-    "libp2p-kad-dht": "~0.16.0",
+    "libp2p-delegated-peer-routing": "^0.4.0",
+    "libp2p-floodsub": "^0.20.0",
+    "libp2p-gossipsub": "^0.2.0",
+    "libp2p-kad-dht": "^0.18.3",
     "libp2p-keychain": "^0.5.2",
-    "libp2p-mdns": "~0.12.0",
+    "libp2p-mdns": "^0.13.0",
     "libp2p-record": "~0.7.0",
-    "libp2p-secio": "~0.11.0",
-    "libp2p-tcp": "^0.13.0",
-    "libp2p-webrtc-star": "~0.16.0",
+    "libp2p-secio": "^0.12.1",
+    "libp2p-tcp": "^0.14.2",
+    "libp2p-webrtc-star": "^0.17.0",
     "libp2p-websocket-star-multi": "~0.4.3",
-    "libp2p-websockets": "~0.12.3",
+    "libp2p-websockets": "^0.13.0",
     "lodash.flatten": "^4.4.0",
     "mafmt": "^6.0.10",
     "merge-options": "^2.0.0",
     "mime-types": "^2.1.21",
     "mkdirp": "~0.5.1",
     "mortice": "^2.0.0",
-    "multiaddr": "^6.1.1",
+    "multiaddr": "^7.2.1",
     "multiaddr-to-uri": "^5.0.0",
     "multibase": "~0.6.0",
     "multicodec": "~0.5.5",
@@ -161,9 +161,9 @@
     "node-fetch": "^2.3.0",
     "p-iteration": "^1.1.8",
     "p-queue": "^6.1.0",
-    "peer-book": "^0.9.1",
-    "peer-id": "~0.12.2",
-    "peer-info": "~0.15.1",
+    "parse-duration": "^0.1.2",
+    "peer-id": "^0.13.5",
+    "peer-info": "^0.17.0",
     "pretty-bytes": "^5.3.0",
     "progress": "^2.0.1",
     "promise-nodeify": "^3.0.1",
@@ -205,9 +205,10 @@
     "execa": "^3.0.0",
     "form-data": "^3.0.0",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.124.1",
+    "interface-ipfs-core": "github:ipfs/interface-js-ipfs-core#refactor/async-iterables",
     "ipfs-interop": "^0.1.1",
-    "ipfsd-ctl": "^0.47.2",
+    "ipfsd-ctl": "github:ipfs/js-ipfsd-ctl#fix/do-not-call-shutdown-twice",
+    "it-all": "^1.0.1",
     "libp2p-websocket-star": "~0.10.2",
     "lodash": "^4.17.15",
     "ncp": "^2.0.0",

src/cli/commands/add.js

@@ -156,20 +156,20 @@ module.exports = {
         ? globSource(argv.file, { recursive: argv.recursive })
         : process.stdin // Pipe directly to ipfs.add
 
-      let finalHash
+      let finalCid
 
       try {
-        for await (const file of ipfs._addAsyncIterator(source, options)) {
+        for await (const file of ipfs.add(source, options)) {
           if (argv.silent) {
             continue
           }
 
           if (argv.quieter) {
-            finalHash = file.hash
+            finalCid = file.cid
             continue
           }
 
-          const cid = cidToString(file.hash, { base: argv.cidBase })
+          const cid = cidToString(file.cid, { base: argv.cidBase })
           let message = cid
 
           if (!argv.quiet) {
@@ -184,7 +184,7 @@ module.exports = {
           bar.terminate()
         }
 
-        // Tweak the error message and add more relevant infor for the CLI
+        // Tweak the error message and add more relevant info for the CLI
         if (err.code === 'ERR_DIR_NON_RECURSIVE') {
           err.message = `'${err.path}' is a directory, use the '-r' flag to specify directories`
         }
@@ -197,7 +197,7 @@ module.exports = {
       }
 
       if (argv.quieter) {
-        log(cidToString(finalHash, { base: argv.cidBase }))
+        log(cidToString(finalCid, { base: argv.cidBase }))
       }
     })())
   }

src/cli/commands/bitswap/stat.js

@@ -35,7 +35,7 @@ module.exports = {
         stats.dupDataReceived = prettyBytes(stats.dupDataReceived.toNumber()).toUpperCase()
         stats.wantlist = `[${stats.wantlist.length} keys]`
       } else {
-        const wantlist = stats.wantlist.map((elem) => cidToString(elem['/'], { base: cidBase, upgrade: false }))
+        const wantlist = stats.wantlist.map(cid => cidToString(cid, { base: cidBase, upgrade: false }))
         stats.wantlist = `[${wantlist.length} keys]
             ${wantlist.join('\n            ')}`
       }

src/cli/commands/bitswap/wantlist.js

@@ -25,7 +25,7 @@ module.exports = {
     resolve((async () => {
       const ipfs = await getIpfs()
       const list = await ipfs.bitswap.wantlist(peer)
-      list.Keys.forEach(k => print(cidToString(k['/'], { base: cidBase, upgrade: false })))
+      list.forEach(cid => print(cidToString(cid, { base: cidBase, upgrade: false })))
     })())
   }
 }

src/cli/commands/block/rm.js

@@ -25,7 +25,7 @@ module.exports = {
       const ipfs = await getIpfs()
       let errored = false
 
-      for await (const result of ipfs.block._rmAsyncIterator(hash, {
+      for await (const result of ipfs.block.rm(hash, {
         force,
         quiet
       })) {

src/cli/commands/block/stat.js

@@ -20,7 +20,7 @@ module.exports = {
     resolve((async () => {
       const ipfs = await getIpfs()
       const stats = await ipfs.block.stat(key)
-      print('Key: ' + cidToString(stats.key, { base: cidBase }))
+      print('Key: ' + cidToString(stats.cid, { base: cidBase }))
       print('Size: ' + stats.size)
     })())
   }

src/cli/commands/dag/resolve.js

@@ -19,10 +19,9 @@ module.exports = {
       const options = {}
 
       try {
-        const result = await ipfs.dag.resolve(ref, options)
         let lastCid
 
-        for (const res of result) {
+        for await (const res of ipfs.dag.resolve(ref, options)) {
           if (CID.isCID(res.value)) {
             lastCid = res.value
           }

src/cli/commands/init.js

@@ -65,7 +65,7 @@ module.exports = {
       const IPFS = require('../../core')
       const Repo = require('ipfs-repo')
 
-      const node = new IPFS({
+      const node = await IPFS.create({
         repo: new Repo(path),
         init: false,
         start: false,

src/cli/commands/pin/add.js

@@ -28,7 +28,7 @@ module.exports = {
       const ipfs = await getIpfs()
       const results = await ipfs.pin.add(ipfsPath, { recursive })
       results.forEach((res) => {
-        print(`pinned ${cidToString(res.hash, { base: cidBase })} ${type}ly`)
+        print(`pinned ${cidToString(res.cid, { base: cidBase })} ${type}ly`)
       })
     })())
   }

src/cli/commands/pin/ls.js

@@ -34,14 +34,13 @@ module.exports = {
     resolve((async () => {
       const paths = ipfsPath
       const ipfs = await getIpfs()
-      const results = await ipfs.pin.ls(paths, { type })
-      results.forEach((res) => {
-        let line = cidToString(res.hash, { base: cidBase })
+      for await (const res of ipfs.pin.ls(paths, { type })) {
+        let line = cidToString(res.cid, { base: cidBase })
         if (!quiet) {
           line += ` ${res.type}`
         }
         print(line)
-      })
+      }
     })())
   }
 }

src/cli/commands/pin/rm.js

@@ -27,7 +27,7 @@ module.exports = {
       const ipfs = await getIpfs()
       const results = await ipfs.pin.rm(ipfsPath, { recursive })
       results.forEach((res) => {
-        print(`unpinned ${cidToString(res.hash, { base: cidBase })}`)
+        print(`unpinned ${cidToString(res.cid, { base: cidBase })}`)
       })
     })())
   }

src/cli/daemon.js

@@ -55,9 +55,7 @@ class Daemon {
 
     // start the daemon
     const ipfsOpts = Object.assign({}, { init: true, start: true, libp2p }, this._options)
-    const ipfs = await IPFS.create(ipfsOpts)
-
-    this._ipfs = ipfs
+    const ipfs = this._ipfs = await IPFS.create(ipfsOpts)
 
     // start HTTP servers (if API or Gateway is enabled in options)
     const httpApi = new HttpApi(ipfs, ipfsOpts)

src/core/api-manager.js

@@ -0,0 +1,21 @@
+module.exports = class ApiManager {
+  constructor () {
+    this._api = {}
+    this._onUndef = () => undefined
+    this.api = new Proxy({}, {
+      get: (_, prop) => {
+        if (prop === 'then') return undefined // Not a promise!
+        return this._api[prop] === undefined ? this._onUndef(prop) : this._api[prop]
+      },
+      has: (_, prop) => prop in this._api
+    })
+  }
+
+  update (nextApi, onUndef) {
+    const prevApi = this._api
+    const prevUndef = this._onUndef
+    this._api = nextApi
+    if (onUndef) this._onUndef = onUndef
+    return { cancel: () => this.update(prevApi, prevUndef), api: this.api }
+  }
+}