From e3688f833539932a60777a334e50754d3f49ccde Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 18 Oct 2017 17:02:57 +0100 Subject: [PATCH 1/5] chore: test against go-ipfs 0.4.11 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7a4577355..97be67c51 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "gulp": "^3.9.1", "interface-ipfs-core": "~0.32.1", "hapi": "^16.6.2", - "ipfsd-ctl": "~0.23.0", + "ipfsd-ctl": "~0.24.0", "pre-commit": "^1.2.2", "socket.io": "^2.0.3", "socket.io-client": "^2.0.3", From 9c34172e769c2cd14e974761f4271c96d093505e Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 6 Sep 2017 14:45:16 +0100 Subject: [PATCH 2/5] feat: avoid doing multiple RPC requests for files.add, fixes #522 --- src/files/add.js | 8 ++-- src/util/fs-add.js | 13 ++---- src/util/url-add.js | 17 +++---- src/utils/{get-dagnode.js => aaa} | 0 src/utils/{dagnode-stream.js => converter.js} | 44 +++++++++---------- src/utils/stream-to-value.js | 4 +- 6 files changed, 35 insertions(+), 51 deletions(-) rename src/utils/{get-dagnode.js => aaa} (100%) rename src/utils/{dagnode-stream.js => converter.js} (53%) diff --git a/src/files/add.js b/src/files/add.js index cfd9a279e..9b305a5bd 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -2,8 +2,8 @@ const isStream = require('is-stream') const promisify = require('promisify-es6') -const DAGNodeStream = require('../utils/dagnode-stream') const ProgressStream = require('../utils/progress-stream') +const ConverterStream = require('../utils/converter') module.exports = (send) => { return promisify((files, opts, callback) => { @@ -44,9 +44,9 @@ module.exports = (send) => { const request = { path: 'add', files: files, qs: qs, progress: opts.progress } - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream - .streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback) + const transform = (res, callback) => { + ConverterStream.streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback) + } send.andTransform(request, transform, callback) }) } diff --git a/src/util/fs-add.js b/src/util/fs-add.js index 0803b36a9..e305069fd 100644 --- a/src/util/fs-add.js +++ b/src/util/fs-add.js @@ -2,7 +2,7 @@ const isNode = require('detect-node') const promisify = require('promisify-es6') -const DAGNodeStream = require('../utils/dagnode-stream') +const converter = require('../utils/converter') const moduleConfig = require('../utils/module-config') module.exports = (arg) => { @@ -31,14 +31,7 @@ module.exports = (arg) => { return callback(new Error('"path" must be a string')) } - const request = { - path: 'add', - qs: opts, - files: path - } - - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) - send.andTransform(request, transform, callback) + const request = { path: 'add', files: path, qs: opts } + send.andTransform(request, converter, callback) }) } diff --git a/src/util/url-add.js b/src/util/url-add.js index 3ecc691d6..a9889e64f 100644 --- a/src/util/url-add.js +++ b/src/util/url-add.js @@ -4,7 +4,7 @@ const promisify = require('promisify-es6') const once = require('once') const parseUrl = require('url').parse const request = require('../utils/request') -const DAGNodeStream = require('../utils/dagnode-stream') +const converter = require('../utils/converter') const moduleConfig = require('../utils/module-config') module.exports = (arg) => { @@ -25,12 +25,12 @@ module.exports = (arg) => { opts = {} } + callback = once(callback) + if (!validUrl(url)) { return callback(new Error('"url" param must be an http(s) url')) } - callback = once(callback) - requestWithRedirect(url, opts, send, callback) }) } @@ -52,14 +52,9 @@ const requestWithRedirect = (url, opts, send, callback) => { } requestWithRedirect(redirection, opts, send, callback) } else { - const params = { - path: 'add', - qs: opts, - files: res - } - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) - send.andTransform(params, transform, callback) + const request = { path: 'add', files: res, qs: opts } + + send.andTransform(request, converter, callback) } }).end() } diff --git a/src/utils/get-dagnode.js b/src/utils/aaa similarity index 100% rename from src/utils/get-dagnode.js rename to src/utils/aaa diff --git a/src/utils/dagnode-stream.js b/src/utils/converter.js similarity index 53% rename from src/utils/dagnode-stream.js rename to src/utils/converter.js index 15198649c..b372e0d09 100644 --- a/src/utils/dagnode-stream.js +++ b/src/utils/converter.js @@ -3,18 +3,18 @@ const pump = require('pump') const TransformStream = require('readable-stream').Transform const streamToValue = require('./stream-to-value') -const getDagNode = require('./get-dagnode') /* Transforms a stream of {Name, Hash} objects to include size of the DAG object. - Usage: inputStream.pipe(DAGNodeStream({ send: send })) + Usage: inputStream.pipe(new Converter()) Input object format: { Name: '/path/to/file/foo.txt', Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP' + Size: '20' } Output object format: @@ -24,38 +24,34 @@ const getDagNode = require('./get-dagnode') size: 20 } */ -class DAGNodeStream extends TransformStream { +class ConverterStream extends TransformStream { constructor (options) { const opts = Object.assign(options || {}, { objectMode: true }) super(opts) - this._send = opts.send } - static streamToValue (send, inputStream, callback) { - const outputStream = pump(inputStream, new DAGNodeStream({ send: send }), (err) => { - if (err) { - callback(err) - } + _transform (obj, enc, callback) { + this.push({ + path: obj.Name, + hash: obj.Hash, + size: parseInt(obj.Size, 10) }) - streamToValue(outputStream, callback) + + callback(null) } +} - _transform (obj, enc, callback) { - getDagNode(this._send, obj.Hash, (err, node) => { +function converter (inputStream, callback) { + const outputStream = pump( + inputStream, + new ConverterStream(), + (err) => { if (err) { - return callback(err) - } - - const result = { - path: obj.Name, - hash: obj.Hash, - size: node.size + callback(err) } - - this.push(result) - callback(null) }) - } + + streamToValue(outputStream, callback) } -module.exports = DAGNodeStream +module.exports = converter diff --git a/src/utils/stream-to-value.js b/src/utils/stream-to-value.js index 69213f776..fa7068d46 100644 --- a/src/utils/stream-to-value.js +++ b/src/utils/stream-to-value.js @@ -6,9 +6,9 @@ const concat = require('concat-stream') /* Concatenate a stream to a single value. */ -function streamToValue (res, callback) { +function streamToValue (response, callback) { pump( - res, + response, concat((data) => callback(null, data)), (err) => { if (err) { From 0150273e5c627fcb819224a7d54fde5a264b97e4 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 6 Sep 2017 15:04:16 +0100 Subject: [PATCH 3/5] new bundle sizes, so much space saved! --- examples/sub-module/bundles-size-KBs.csv | 54 ++++++++++++------------ examples/sub-module/test-modules-size.sh | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) mode change 100644 => 100755 examples/sub-module/test-modules-size.sh diff --git a/examples/sub-module/bundles-size-KBs.csv b/examples/sub-module/bundles-size-KBs.csv index 8101c63cd..3bc5bce32 100644 --- a/examples/sub-module/bundles-size-KBs.csv +++ b/examples/sub-module/bundles-size-KBs.csv @@ -1,28 +1,28 @@ name, bundled (KBs), minified (KBs) -IPFS, 1685.41, 661.64 -add, 968.11, 344.13 -bitswap, 684.80, 232.13 -block, 724.77, 250.23 -bootstrap, 685.64, 232.39 -cat, 725.05, 250.33 -commands, 683.89, 231.76 -config, 686.87, 233.10 -dht, 688.51, 233.41 -diag, 684.97, 232.18 -files, 1120.38, 404.30 -get, 907.74, 318.35 -id, 684.31, 231.95 -key, 684.59, 232.03 -log, 685.40, 232.38 -ls, 684.00, 231.80 -mount, 684.18, 231.86 -name, 684.63, 232.06 -object, 923.66, 340.57 -pin, 685.51, 232.39 -ping, 684.59, 231.90 -pubsub, 740.40, 249.54 -refs, 684.39, 231.94 -repo, 684.56, 232.02 -swarm, 1324.18, 527.03 -update, 684.45, 231.96 -version, 684.21, 231.88 +IPFS, 1412.20, 573.44 +add, 591.17, 198.23 +bitswap, 590.14, 197.96 +block, 630.50, 216.31 +bootstrap, 590.97, 198.22 +cat, 630.78, 216.41 +commands, 589.22, 197.59 +config, 592.21, 198.93 +dht, 593.86, 199.24 +diag, 590.31, 198.00 +files, 669.07, 235.88 +get, 661.57, 233.16 +id, 589.65, 197.78 +key, 589.93, 197.86 +log, 590.74, 198.20 +ls, 589.35, 197.63 +mount, 589.53, 197.69 +name, 589.97, 197.88 +object, 833.17, 307.73 +pin, 590.86, 198.22 +ping, 589.94, 197.73 +pubsub, 595.31, 199.76 +refs, 589.74, 197.77 +repo, 589.91, 197.85 +swarm, 1239.42, 498.59 +update, 589.79, 197.79 +version, 589.55, 197.71 diff --git a/examples/sub-module/test-modules-size.sh b/examples/sub-module/test-modules-size.sh old mode 100644 new mode 100755 index ef9955438..0b330d6a2 --- a/examples/sub-module/test-modules-size.sh +++ b/examples/sub-module/test-modules-size.sh @@ -6,7 +6,7 @@ modules=($(ls modules/)) echo "name, bundled (KBs), minified (KBs)" # Full IPFS module -webpack --display none --config webpack.confg.js complete-module.js complete-bundle.js +webpack --display none --config webpack.config.js complete-module.js complete-bundle.js babili complete-bundle.js -o complete-bundle-minified.js ipfsBundleSize=($(wc -c < complete-bundle.js | awk '{b=$1/1024; printf "%.2f\n", b}' | sed 's/,/./g')) From a596668abb2eb989abb9cfa809488e4addd38d9e Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 18 Oct 2017 17:23:11 +0100 Subject: [PATCH 4/5] use converter --- src/files/add.js | 10 +++++---- src/utils/aaa | 54 ------------------------------------------------ 2 files changed, 6 insertions(+), 58 deletions(-) delete mode 100644 src/utils/aaa diff --git a/src/files/add.js b/src/files/add.js index 9b305a5bd..4d8b07220 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -2,8 +2,8 @@ const isStream = require('is-stream') const promisify = require('promisify-es6') -const ProgressStream = require('../utils/progress-stream') -const ConverterStream = require('../utils/converter') +// const ProgressStream = require('../utils/progress-stream') +const converter = require('../utils/converter') module.exports = (send) => { return promisify((files, opts, callback) => { @@ -44,9 +44,11 @@ module.exports = (send) => { const request = { path: 'add', files: files, qs: qs, progress: opts.progress } + send.andTransform(request, converter, callback) + /* const transform = (res, callback) => { - ConverterStream.streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback) + converter(send, ProgressStream.fromStream(opts.progress, res), callback) } - send.andTransform(request, transform, callback) + */ }) } diff --git a/src/utils/aaa b/src/utils/aaa deleted file mode 100644 index 1cfd1dcd7..000000000 --- a/src/utils/aaa +++ /dev/null @@ -1,54 +0,0 @@ -'use strict' - -const DAGNode = require('ipld-dag-pb').DAGNode -const parallel = require('async/parallel') -const CID = require('cids') -const streamToValue = require('./stream-to-value') - -module.exports = function (send, hash, callback) { - let cid - - try { - cid = new CID(hash) - } catch (err) { - return callback(err) - } - - // Retrieve the object and its data in parallel, then produce a DAGNode - // instance using this information. - parallel([ - (done) => { - send({ - path: 'object/get', - args: cid.toBaseEncodedString() - }, done) - }, - (done) => { - // WORKAROUND: request the object's data separately, since raw bits in JSON - // are interpreted as UTF-8 and corrupt the data. - // See https://github.com/ipfs/go-ipfs/issues/1582 for more details. - send({ - path: 'object/data', - args: cid.toBaseEncodedString() - }, done) - } - ], (err, res) => { - if (err) { - return callback(err) - } - - var object = res[0] - var stream = res[1] - - if (Buffer.isBuffer(stream)) { - DAGNode.create(stream, object.Links, callback) - } else { - streamToValue(stream, (err, data) => { - if (err) { - return callback(err) - } - DAGNode.create(data, object.Links, callback) - }) - } - }) -} From f79a610b76565aa6580a15265468f987683f8f2d Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 18 Oct 2017 17:28:58 +0100 Subject: [PATCH 5/5] add progress tracking back --- src/files/add.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/files/add.js b/src/files/add.js index 4d8b07220..324a49416 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -2,7 +2,7 @@ const isStream = require('is-stream') const promisify = require('promisify-es6') -// const ProgressStream = require('../utils/progress-stream') +const ProgressStream = require('../utils/progress-stream') const converter = require('../utils/converter') module.exports = (send) => { @@ -44,11 +44,8 @@ module.exports = (send) => { const request = { path: 'add', files: files, qs: qs, progress: opts.progress } - send.andTransform(request, converter, callback) - /* - const transform = (res, callback) => { - converter(send, ProgressStream.fromStream(opts.progress, res), callback) - } - */ + send.andTransform(request, (response, cb) => { + converter(ProgressStream.fromStream(opts.progress, response), cb) + }, callback) }) }