From 7a9aa4e2c16fdb6314da116d848048a81ab427ac Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 20 Apr 2018 17:04:50 +0100 Subject: [PATCH 1/6] fix: files.add with pull streams --- package.json | 1 + src/core/components/files.js | 81 +++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index fb7720e60f..c8208502e0 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "ipfs-unixfs-engine": "~0.29.0", "ipld": "^0.17.0", "is-ipfs": "^0.3.2", + "is-pull-stream": "0.0.0", "is-stream": "^1.1.0", "joi": "^13.2.0", "joi-browser": "^13.0.1", diff --git a/src/core/components/files.js b/src/core/components/files.js index c1a8f4cb7b..1df23848a4 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -12,6 +12,7 @@ const toPull = require('stream-to-pull-stream') const deferred = require('pull-defer') const waterfall = require('async/waterfall') const isStream = require('is-stream') +const isSource = require('is-pull-stream').isSource const Duplex = require('readable-stream').Duplex const OtherBuffer = require('buffer').Buffer const CID = require('cids') @@ -60,6 +61,10 @@ function normalizeContent (opts, content) { data = { path: '', content: toPull.source(data) } } + if (isSource(data)) { + data = { path: '', content: data } + } + if (data && data.content && typeof data.content !== 'function') { if (Buffer.isBuffer(data.content)) { data.content = pull.values([data.content]) @@ -196,40 +201,56 @@ module.exports = function files (self) { } return { - add: promisify((data, options = {}, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } else if (!callback || typeof callback !== 'function') { - callback = noop - } + add: (() => { + const add = promisify((data, options = {}, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } else if (!callback || typeof callback !== 'function') { + callback = noop + } - const ok = Buffer.isBuffer(data) || - isStream.readable(data) || - Array.isArray(data) || - OtherBuffer.isBuffer(data) || - typeof data === 'object' + const ok = Buffer.isBuffer(data) || + isStream.readable(data) || + Array.isArray(data) || + OtherBuffer.isBuffer(data) || + typeof data === 'object' || + isSource(data) - if (!ok) { - return callback(new Error('first arg must be a buffer, readable stream, an object or array of objects')) - } + if (!ok) { + return callback(new Error('first arg must be a buffer, readable stream, pull stream, an object or array of objects')) + } - // CID v0 is for multihashes encoded with sha2-256 - if (options.hashAlg && options.cidVersion !== 1) { - options.cidVersion = 1 - } + // CID v0 is for multihashes encoded with sha2-256 + if (options.hashAlg && options.cidVersion !== 1) { + options.cidVersion = 1 + } - pull( - pull.values([data]), - _addPullStream(options), - sort((a, b) => { - if (a.path < b.path) return 1 - if (a.path > b.path) return -1 - return 0 - }), - pull.collect(callback) - ) - }), + pull( + pull.values([data]), + _addPullStream(options), + sort((a, b) => { + if (a.path < b.path) return 1 + if (a.path > b.path) return -1 + return 0 + }), + pull.collect(callback) + ) + }) + + return function () { + const args = Array.from(arguments) + + // If we files.add(), then promisify thinks the pull stream + // is a callback! Add an empty options object in this case so that a + // promise is returned. + if (args.length === 1 && isSource(args[0])) { + args.push({}) + } + + return add.apply(null, args) + } + })(), addReadableStream: (options) => { options = options || {} From 89300d4a5b02ecd4bd6d5c3c5246e94f5950d932 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 30 Apr 2018 21:01:49 +0100 Subject: [PATCH 2/6] chore: update deps --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index c8208502e0..574a25dead 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "expose-loader": "^0.7.5", "form-data": "^2.3.2", "hat": "0.0.3", - "interface-ipfs-core": "^0.61.0", + "interface-ipfs-core": "^0.64.2", "ipfsd-ctl": "^0.32.1", "lodash": "^4.17.10", "mocha": "^5.1.1", @@ -162,7 +162,6 @@ "pull-zip": "^2.0.1", "read-pkg-up": "^3.0.0", "readable-stream": "2.3.6", - "safe-buffer": "^5.1.2", "stream-to-pull-stream": "^1.7.2", "tar-stream": "^1.6.0", "temp": "~0.8.3", From 60e7f58cd8e72d6f8bd8afbd26faf91f23f1a9de Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 30 Apr 2018 22:07:32 +0200 Subject: [PATCH 3/6] feat: Provide access to bundled libraries when in browser (#1297) --- README.md | 10 ++++++++++ package.json | 3 +++ src/core/index.js | 16 +++++++++++++++- test/core/init.spec.js | 17 ++++++++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17ba032aca..ed868c5e94 100644 --- a/README.md +++ b/README.md @@ -459,8 +459,18 @@ A set of data types are exposed directly from the IPFS instance under `ipfs.type - [`ipfs.types.PeerId`](https://github.com/libp2p/js-peer-id) - [`ipfs.types.PeerInfo`](https://github.com/libp2p/js-peer-info) - [`ipfs.types.multiaddr`](https://github.com/multiformats/js-multiaddr) +- [`ipfs.types.multibase`](https://github.com/multiformats/multibase) - [`ipfs.types.multihash`](https://github.com/multiformats/js-multihash) - [`ipfs.types.CID`](https://github.com/ipld/js-cid) +- [`ipfs.types.dagPB`](https://github.com/ipld/js-ipld-dag-pb) +- [`ipfs.types.dagCBOR`](https://github.com/ipld/js-ipld-dag-cbor) + +#### `Util` + +A set of utils are exposed directly from the IPFS instance under `ipfs.util`. That way you're not required to import/require the following: + +- [`ipfs.util.crypto`](https://github.com/libp2p/js-libp2p-crypto) +- [`ipfs.util.isIPFS`](https://github.com/ipfs-shipyard/is-ipfs) ## FAQ diff --git a/package.json b/package.json index 574a25dead..e572ea6ae1 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,8 @@ "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.20.0", "ipfs-unixfs": "~0.1.14", + "ipld-dag-cbor": "^0.12.0", + "ipld-dag-pb": "^0.13.1", "ipfs-unixfs-engine": "~0.29.0", "ipld": "^0.17.0", "is-ipfs": "^0.3.2", @@ -141,6 +143,7 @@ "mafmt": "^6.0.0", "mime-types": "^2.1.18", "mkdirp": "~0.5.1", + "multibase": "^0.4.0", "multiaddr": "^5.0.0", "multihashes": "~0.4.13", "once": "^1.4.0", diff --git a/src/core/index.js b/src/core/index.js index d27d178f85..46f8c9bb2c 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -4,9 +4,14 @@ const BlockService = require('ipfs-block-service') const Ipld = require('ipld') const PeerId = require('peer-id') const PeerInfo = require('peer-info') +const dagCBOR = require('ipld-dag-cbor') +const dagPB = require('ipld-dag-pb') +const crypto = require('libp2p-crypto') +const isIPFS = require('is-ipfs') const multiaddr = require('multiaddr') const multihash = require('multihashes') const PeerBook = require('peer-book') +const multibase = require('multibase') const CID = require('cids') const debug = require('debug') const extend = require('deep-extend') @@ -58,8 +63,11 @@ class IPFS extends EventEmitter { PeerId: PeerId, PeerInfo: PeerInfo, multiaddr: multiaddr, + multibase: multibase, multihash: multihash, - CID: CID + CID: CID, + dagPB: dagPB, + dagCBOR: dagCBOR } // IPFS Core Internals @@ -120,6 +128,12 @@ class IPFS extends EventEmitter { this.lsReadableStream = this.files.lsReadableStreamImmutable this.lsPullStream = this.files.lsPullStreamImmutable + // ipfs.util + this.util = { + crypto: crypto, + isIPFS: isIPFS + } + boot(this) } } diff --git a/test/core/init.spec.js b/test/core/init.spec.js index 58916ecbe4..92c4623d97 100644 --- a/test/core/init.spec.js +++ b/test/core/init.spec.js @@ -10,7 +10,12 @@ const isNode = require('detect-node') const hat = require('hat') const PeerId = require('peer-id') const PeerInfo = require('peer-info') +const dagCBOR = require('ipld-dag-cbor') +const dagPB = require('ipld-dag-pb') +const crypto = require('libp2p-crypto') +const isIPFS = require('is-ipfs') const multiaddr = require('multiaddr') +const multibase = require('multibase') const multihash = require('multihashes') const CID = require('cids') const IPFS = require('../../src/core') @@ -101,8 +106,18 @@ describe('init', () => { PeerId: PeerId, PeerInfo: PeerInfo, multiaddr: multiaddr, + multibase: multibase, multihash: multihash, - CID: CID + CID: CID, + dagPB: dagPB, + dagCBOR: dagCBOR + }) + }) + + it('util', () => { + expect(ipfs.util).to.be.deep.equal({ + crypto: crypto, + isIPFS: isIPFS }) }) }) From 6248cac3c02b4ff4fc8399e8cd4a2a35e150e091 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 30 Apr 2018 21:10:58 +0100 Subject: [PATCH 4/6] chore: update deps --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e572ea6ae1..a9e8dde17c 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "ipfs-repo": "~0.20.0", "ipfs-unixfs": "~0.1.14", "ipld-dag-cbor": "^0.12.0", - "ipld-dag-pb": "^0.13.1", + "ipld-dag-pb": "^0.14.4", "ipfs-unixfs-engine": "~0.29.0", "ipld": "^0.17.0", "is-ipfs": "^0.3.2", From c1fd9763f6d1dc1636babedf9a830a6e6de876d7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 20 Feb 2018 18:36:03 +0000 Subject: [PATCH 5/6] docs: Add browser example for ReadableStreams feat: Allows for byte offsets when using ipfs.files.cat and friends to request slices of files --- examples/README.md | 1 + examples/browser-readablestream/README.md | 16 ++ examples/browser-readablestream/index.html | 66 ++++++++ examples/browser-readablestream/index.js | 75 +++++++++ examples/browser-readablestream/package.json | 22 +++ examples/browser-readablestream/utils.js | 145 ++++++++++++++++++ .../browser-readablestream/webpack.config.js | 29 ++++ package.json | 3 +- src/cli/commands/files/cat.js | 20 ++- src/core/components/files.js | 47 ++++-- src/http/api/resources/files.js | 21 ++- test/cli/files.js | 18 +++ 12 files changed, 445 insertions(+), 18 deletions(-) create mode 100644 examples/browser-readablestream/README.md create mode 100644 examples/browser-readablestream/index.html create mode 100644 examples/browser-readablestream/index.js create mode 100644 examples/browser-readablestream/package.json create mode 100644 examples/browser-readablestream/utils.js create mode 100644 examples/browser-readablestream/webpack.config.js diff --git a/examples/README.md b/examples/README.md index b4678c1fa2..e47ef446ec 100644 --- a/examples/README.md +++ b/examples/README.md @@ -22,6 +22,7 @@ Let us know if you find any issue or if you want to contribute and add a new tut - [js-ipfs in electron](./run-in-electron) - [Using streams to add a directory of files to ipfs](./browser-add-readable-stream) - [Customizing the ipfs repository](./custom-ipfs-repo) +- - [Streaming video from ipfs to the browser using `ReadableStream`s](./browser-readablestream) ## Understanding the IPFS Stack diff --git a/examples/browser-readablestream/README.md b/examples/browser-readablestream/README.md new file mode 100644 index 0000000000..6fe75111f8 --- /dev/null +++ b/examples/browser-readablestream/README.md @@ -0,0 +1,16 @@ +# Streaming video from IPFS using ReadableStreams + +We can use the execllent [`videostream`](https://www.npmjs.com/package/videostream) to stream video from IPFS to the browser. All we need to do is return a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream)-like object that contains the requested byte ranges. + +Take a look at [`index.js`](./index.js) to see a working example. + +## Running the demo + +In this directory: + +``` +$ npm install +$ npm start +``` + +Then open [http://localhost:8888](http://localhost:8888) in your browser. diff --git a/examples/browser-readablestream/index.html b/examples/browser-readablestream/index.html new file mode 100644 index 0000000000..61ded58b13 --- /dev/null +++ b/examples/browser-readablestream/index.html @@ -0,0 +1,66 @@ + + + + + <%= htmlWebpackPlugin.options.title %> + + + +
+
+
+ + +
+ +
+ 

+    

+  
+
diff --git a/examples/browser-readablestream/index.js b/examples/browser-readablestream/index.js
new file mode 100644
index 0000000000..7a2a969fa1
--- /dev/null
+++ b/examples/browser-readablestream/index.js
@@ -0,0 +1,75 @@
+'use strict'
+
+/* eslint-env browser */
+
+const Ipfs = require('../../')
+const videoStream = require('videostream')
+const ipfs = new Ipfs({ repo: 'ipfs-' + Math.random() })
+const {
+  dragDrop,
+  statusMessages,
+  createVideoElement,
+  log
+} = require('./utils')
+
+log('IPFS: Initialising')
+
+ipfs.on('ready', () => {
+  // Set up event listeners on the