From 69681a7d7a8434c11f6f10e370e324f5a3d31042 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 6 Nov 2020 14:18:35 +0000 Subject: [PATCH] feat: pass file name to add/addAll progress handler (#3372) Since ipfs/js-ipfs-unixfs#87 landed we can now pass the file name to the progress handler for adding files: ```js await ipfs.addAll(..., { progress: (bytes, fileName) => { //... } }) ``` This should make showing progress a bit more usable. Co-authored-by: Hugo Dias --- docs/core-api/FILES.md | 2 +- packages/interface-ipfs-core/src/add-all.js | 20 +++++++ packages/interface-ipfs-core/src/add.js | 14 +++++ .../ipfs-core/src/components/add-all/index.js | 8 +-- packages/ipfs-http-client/src/add-all.js | 2 +- packages/ipfs-http-client/src/index.js | 4 +- packages/ipfs-http-client/src/interface.ts | 4 +- .../ipfs-message-port-client/package.json | 1 + packages/ipfs-message-port-client/src/core.js | 45 +------------- .../ipfs-message-port-client/src/index.js | 36 ++++++++++++ .../ipfs-message-port-client/src/interface.ts | 58 +++++++++++++++++++ .../ipfs-message-port-client/tsconfig.json | 6 ++ .../ipfs-message-port-protocol/src/core.js | 18 +++--- .../test/core.browser.js | 8 +-- packages/ipfs-message-port-server/src/core.js | 23 ++++++-- 15 files changed, 178 insertions(+), 71 deletions(-) create mode 100644 packages/ipfs-message-port-client/src/interface.ts diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index 68b114193f..b164cef339 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -252,7 +252,7 @@ An optional object which may have the following keys: | hashAlg | `String` | `'sha2-256'` | multihash hashing algorithm to use | | onlyHash | `boolean` | `false` | If true, will not add blocks to the blockstore | | pin | `boolean` | `true` | pin this object when adding | -| progress | function | `undefined` | a function that will be called with the byte length of chunks as a file is added to ipfs | +| progress | function | `undefined` | a function that will be called with the number of bytes added as a file is added to ipfs and the name of the file being added | | rawLeaves | `boolean` | `false` | if true, DAG leaves will contain raw file data and not be wrapped in a protobuf | | shardSplitThreshold | `Number` | `1000` | Directories with more than this number of files will be created as HAMT-sharded directories | | trickle | `boolean` | `false` | if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation | diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 5c53621fcf..46887d3af7 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -170,6 +170,26 @@ module.exports = (common, options) => { expect(root.cid.toString()).to.equal(fixtures.directory.cid) }) + it('should receive file name from progress event', async () => { + const receivedNames = [] + function handler (p, name) { + receivedNames.push(name) + } + + await drain(ipfs.addAll([{ + content: 'hello', + path: 'foo.txt' + }, { + content: 'world', + path: 'bar.txt' + }], { + progress: handler, + wrapWithDirectory: true + })) + + expect(receivedNames).to.deep.equal(['foo.txt', 'bar.txt']) + }) + it('should add files to a directory non sequentially', async function () { const content = path => ({ path: `test-dir/${path}`, diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index fdf27d281e..f6e2c6a82a 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -130,6 +130,20 @@ module.exports = (common, options) => { expect(accumProgress).to.equal(fixtures.emptyFile.data.length) }) + it('should receive file name from progress event', async () => { + let receivedName + function handler (p, name) { + receivedName = name + } + + await ipfs.add({ + content: 'hello', + path: 'foo.txt' + }, { progress: handler }) + + expect(receivedName).to.equal('foo.txt') + }) + it('should add an empty file without progress enabled', async () => { const file = await ipfs.add(fixtures.emptyFile.data) diff --git a/packages/ipfs-core/src/components/add-all/index.js b/packages/ipfs-core/src/components/add-all/index.js index c5c7502ddb..f383304f15 100644 --- a/packages/ipfs-core/src/components/add-all/index.js +++ b/packages/ipfs-core/src/components/add-all/index.js @@ -47,9 +47,9 @@ module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) let total = 0 const prog = opts.progress - opts.progress = (bytes) => { + opts.progress = (bytes, fileName) => { total += bytes - prog(total) + prog(total, fileName) } } @@ -162,8 +162,8 @@ function pinFile (pin, opts) { * @property {boolean} [onlyHash=false] - If true, will not add blocks to the * blockstore. * @property {boolean} [pin=true] - Pin this object when adding. - * @property {(bytes:number) => void} [progress] - A function that will be - * called with the byte length of chunks as a file is added to ipfs. + * @property {(bytes:number, fileName:string) => void} [progress] - A function that will be + * called with the number of bytes added as a file is added to ipfs and the name of the file being added. * @property {boolean} [rawLeaves=false] - If true, DAG leaves will contain raw * file data and not be wrapped in a protobuf. * @property {number} [shardSplitThreshold=1000] - Directories with more than this diff --git a/packages/ipfs-http-client/src/add-all.js b/packages/ipfs-http-client/src/add-all.js index ba226c8570..0ef589ee8a 100644 --- a/packages/ipfs-http-client/src/add-all.js +++ b/packages/ipfs-http-client/src/add-all.js @@ -38,7 +38,7 @@ module.exports = configure((api) => { if (file.hash !== undefined) { yield toCoreInterface(file) } else if (progressFn) { - progressFn(file.bytes || 0) + progressFn(file.bytes || 0, file.name) } } } diff --git a/packages/ipfs-http-client/src/index.js b/packages/ipfs-http-client/src/index.js index 65e7d7c482..11c2ad9d9f 100644 --- a/packages/ipfs-http-client/src/index.js +++ b/packages/ipfs-http-client/src/index.js @@ -68,7 +68,7 @@ module.exports = ipfsClient * derives API from it's return type and extends it last `options` parameter * with `HttpOptions`. * - * This can be used to avoid (re)typing API interface when implemeting it in + * This can be used to avoid (re)typing API interface when implementing it in * http client e.g you can annotate `ipfs.addAll` implementation with * * `@type {Implements}` @@ -83,7 +83,7 @@ module.exports = ipfsClient /** * @template Key * @template {(config:any) => any} APIFactory - * @typedef {import('./interface').APIMethadWithExtraOptions, Key, HttpOptions>} ImplementsMethod + * @typedef {import('./interface').APIMethodWithExtraOptions, Key, HttpOptions>} ImplementsMethod */ /** diff --git a/packages/ipfs-http-client/src/interface.ts b/packages/ipfs-http-client/src/interface.ts index 34b698cc9f..6258fbd835 100644 --- a/packages/ipfs-http-client/src/interface.ts +++ b/packages/ipfs-http-client/src/interface.ts @@ -1,5 +1,5 @@ // This file contains some utility types that either can't be expressed in -// JSDoc syntax or that result in a different behavior when typed in JSDoc. +// JSDoc syntax or that result in a different behaviour when typed in JSDoc. /** * Utility type that takes IPFS Core API function type (with 0 to 4 arguments @@ -51,7 +51,7 @@ type WithExtendedOptions = Params extends [...End] ? [a1?: A1, a2?: A2, a3?: A3, options?: Options & Ext] : never -export type APIMethadWithExtraOptions < +export type APIMethodWithExtraOptions < API, Key extends keyof API, Extra diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index f91921b1c9..950b85e78f 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -50,6 +50,7 @@ "cross-env": "^7.0.0", "interface-ipfs-core": "^0.141.0", "ipfs": "^0.51.0", + "ipfs-core": "^0.1.0", "ipfs-message-port-protocol": "^0.3.0", "ipfs-message-port-server": "^0.3.0", "ipld-dag-pb": "^0.20.0", diff --git a/packages/ipfs-message-port-client/src/core.js b/packages/ipfs-message-port-client/src/core.js index f2f096ca8c..cddac962a2 100644 --- a/packages/ipfs-message-port-client/src/core.js +++ b/packages/ipfs-message-port-client/src/core.js @@ -73,30 +73,7 @@ class CoreClient extends Client { * `transfer: [input.buffer]` which would allow transferring it instead of * copying. * - * @param {AddAllInput} input - * @param {Object} [options] - * @param {string} [options.chunker="size-262144"] - * @param {number} [options.cidVersion=0] - * @param {boolean} [options.enableShardingExperiment] - * @param {string} [options.hashAlg="sha2-256"] - * @param {boolean} [options.onlyHash=false] - * @param {boolean} [options.pin=true] - * @param {function(number):void} [options.progress] - * @param {boolean} [options.rawLeaves=false] - * @param {number} [options.shardSplitThreshold=1000] - * @param {boolean} [options.trickle=false] - * @param {boolean} [options.wrapWithDirectory=false] - * @param {number} [options.timeout] - * @param {Transferable[]} [options.transfer] - * @param {AbortSignal} [options.signal] - * @returns {AsyncIterable} - * - * @typedef {Object} AddedData - * @property {string} path - * @property {CID} cid - * @property {number} mode - * @property {number} size - * @property {Time} mtime + * @type {import('.').Implements} */ async * addAll (input, options = {}) { const { timeout, signal } = options @@ -123,23 +100,7 @@ class CoreClient extends Client { * `transfer: [input.buffer]` which would allow transferring it instead of * copying. * - * @param {AddInput} input - * @param {Object} [options] - * @param {string} [options.chunker="size-262144"] - * @param {number} [options.cidVersion=0] - * @param {boolean} [options.enableShardingExperiment] - * @param {string} [options.hashAlg="sha2-256"] - * @param {boolean} [options.onlyHash=false] - * @param {boolean} [options.pin=true] - * @param {function(number):void} [options.progress] - * @param {boolean} [options.rawLeaves=false] - * @param {number} [options.shardSplitThreshold=1000] - * @param {boolean} [options.trickle=false] - * @param {boolean} [options.wrapWithDirectory=false] - * @param {number} [options.timeout] - * @param {Transferable[]} [options.transfer] - * @param {AbortSignal} [options.signal] - * @returns {Promise} + * @type {import('.').Implements} */ async add (input, options = {}) { const { timeout, signal } = options @@ -200,7 +161,7 @@ class CoreClient extends Client { * Decodes values yield by `ipfs.add`. * * @param {AddedEntry} data - * @returns {AddedData} + * @returns {import('ipfs-core/src/components/add-all').UnixFSEntry} */ const decodeAddedData = ({ path, cid, mode, mtime, size }) => { return { diff --git a/packages/ipfs-message-port-client/src/index.js b/packages/ipfs-message-port-client/src/index.js index fe005187e2..adfa0c8ae9 100644 --- a/packages/ipfs-message-port-client/src/index.js +++ b/packages/ipfs-message-port-client/src/index.js @@ -62,3 +62,39 @@ class IPFSClient extends CoreClient { } module.exports = IPFSClient + +/** + * @typedef {Object} MessagePortOptions + * @property {Array} [transfer] - A list of ArrayBuffers whose ownership will be transferred to the shared worker + * + * @typedef {import('ipfs-core/src/utils').AbortOptions} AbortOptions} + */ + +/** + * This is an utility type that can be used to derive type of the HTTP Client + * API from the Core API. It takes type of the API factory (from ipfs-core), + * derives API from it's return type and extends it last `options` parameter + * with `HttpOptions`. + * + * This can be used to avoid (re)typing API interface when implementing it in + * http client e.g you can annotate `ipfs.addAll` implementation with + * + * `@type {Implements}` + * + * **Caution**: This supports APIs with up to four parameters and last optional + * `options` parameter, anything else will result to `never` type. + * + * @template {(config:any) => any} APIFactory + * @typedef {APIWithExtraOptions, MessagePortOptions>} Implements + */ + +/** + * @template Key + * @template {(config:any) => any} APIFactory + * @typedef {import('./interface').APIMethodWithExtraOptions, Key, MessagePortOptions>} ImplementsMethod + */ + +/** + * @template API, Extra + * @typedef {import('./interface').APIWithExtraOptions} APIWithExtraOptions + */ diff --git a/packages/ipfs-message-port-client/src/interface.ts b/packages/ipfs-message-port-client/src/interface.ts new file mode 100644 index 0000000000..6258fbd835 --- /dev/null +++ b/packages/ipfs-message-port-client/src/interface.ts @@ -0,0 +1,58 @@ +// This file contains some utility types that either can't be expressed in +// JSDoc syntax or that result in a different behaviour when typed in JSDoc. + +/** + * Utility type that takes IPFS Core API function type (with 0 to 4 arguments + * & last **optional** `options` parameter) and derives a function type with + * `options` parameter extended with given `Extra` options. + * + * **Caution**: API Functions with more than for arguments ahead of `options` + * will result to `never` type. API function that does not take `options` will + * result in function whose last argument is extended with `Extra` which would + * be an error. + */ +// This is typed in TS file because otherwise TS unifies on the first parameter +// regardless of number of parameters function has. +export type APIWithExtraOptions any, Extra> = + (...args: WithExtendedOptions, Extra>) => ReturnType + +type End = never[] +type WithExtendedOptions = Params extends [...End] + ? [] + // (options?: Options) -> (options?: Options & Ext) + : Params extends [options?: infer Options, ...end: End] + ? [options?: Options & Ext] + // (a: A1, options?: Options) -> (a1: A1, options?: Options & Ext) + : Params extends [a1: infer A1, options?: infer Options, ...end: End] + ? [a1: A1, options?: Options & Ext] + // (a1?: A1, options?: Options) -> (a1?: A1, options?: Options & Ext) + : Params extends [a1?: infer A1, options?: infer Options, ...end: End] + ? [a1?: A1, options?: Options & Ext] + // (a1: A1, a2: A2, options?: Options) -> (a1: A1, a2: A2 options?: Options & Ext) + : Params extends [a1: infer A1, a2: infer A2, options?: infer Options, ...end: End] + ? [a1: A1, a2: A2, options?: Options & Ext] + // (a1: A1, a2?: A2, options?: Options) -> (a1: A1, a2?: A2 options?: Options & Ext) + : Params extends [a1: infer A1, a2?: infer A2, options?: infer Options, ...end: End] + ? [a1: A1, a2?: A2, options?: Options & Ext] + // (a1: A1, a2?: A2, options?: Options) -> (a1: A1, a2?: A2 options?: Options & Ext) + : Params extends [a1?: infer A1, a2?: infer A2, options?: infer Options, ...end: End] + ? [a1?: A1, a2?: A2, options?: Options & Ext] + // (a1: A1, a2: A2, a3:A3 options?: Options) -> (a1: A1, a2: A2, a3:A3, options?: Options & Ext) + : Params extends [a1: infer A1, a2: infer A2, a3:infer A3, options?: infer Options, ...end: End] + ? [a1: A1, a2: A2, a3: A3, options?: Options & Ext] + // (a1: A1, a2: A2, a3?:A3 options?: Options) -> (a1: A1, a2: A2, a3?:A3, options?: Options & Ext) + : Params extends [a1: infer A1, a2:infer A2, a3?: infer A3, options?: infer Options, ...end: End] + ? [a1: A1, a2: A2, a3?: A3, options?: Options & Ext] + // (a1: A1, a2?: A2, a3?:A3 options?: Options) -> (a1: A1, a2?: A2, a3?:A3, options?: Options & Ext) + : Params extends [a1: infer A1, a2?: infer A2, a3?: infer A3, options?: infer Options, ...end: End] + ? [a1: A1, a2?: A2, a3?: A3, options?: Options & Ext] + // (a1?: A1, a2?: A2, a3?:A3 options?: Options) -> (a1?: A1, a2?: A2, a3?:A3, options?: Options & Ext) + : Params extends [a1?: infer A1, a2?: infer A2, a3?: infer A3, options?: infer Options, ...end: End] + ? [a1?: A1, a2?: A2, a3?: A3, options?: Options & Ext] + : never + +export type APIMethodWithExtraOptions < + API, + Key extends keyof API, + Extra + > = API[Key] extends (...args: any[]) => any ? APIWithExtraOptions : never diff --git a/packages/ipfs-message-port-client/tsconfig.json b/packages/ipfs-message-port-client/tsconfig.json index 600c8308cd..ad2457a225 100644 --- a/packages/ipfs-message-port-client/tsconfig.json +++ b/packages/ipfs-message-port-client/tsconfig.json @@ -13,6 +13,12 @@ }, { "path": "../ipfs-message-port-server" + }, + { + "path": "../ipfs-core" + }, + { + "path": "../ipfs-core-utils" } ] } diff --git a/packages/ipfs-message-port-protocol/src/core.js b/packages/ipfs-message-port-protocol/src/core.js index 56e7552978..b32559baff 100644 --- a/packages/ipfs-message-port-protocol/src/core.js +++ b/packages/ipfs-message-port-protocol/src/core.js @@ -11,7 +11,6 @@ const { encodeError, decodeError } = require('./error') */ /** - * @template T * @typedef {Object} RemoteCallback * @property {'RemoteCallback'} type * @property {MessagePort} port @@ -173,15 +172,14 @@ const toIterator = iterable => { } /** - * @template T - * @param {function(T):void} callback + * @param {Function} callback * @param {Transferable[]} transfer - * @returns {RemoteCallback} + * @returns {RemoteCallback} */ const encodeCallback = (callback, transfer) => { // eslint-disable-next-line no-undef const { port1: port, port2: remote } = new MessageChannel() - port.onmessage = ({ data }) => callback(data) + port.onmessage = ({ data }) => callback.apply(null, data) transfer.push(remote) return { type: 'RemoteCallback', port: remote } } @@ -189,17 +187,17 @@ exports.encodeCallback = encodeCallback /** * @template T - * @param {RemoteCallback} remote - * @returns {function(T):void | function(T, Transferable[]):void} + * @param {RemoteCallback} remote + * @returns {function(T[]):void | function(T[], Transferable[]):void} */ const decodeCallback = ({ port }) => { /** - * @param {T} value + * @param {T[]} args * @param {Transferable[]} [transfer] * @returns {void} */ - const callback = (value, transfer = []) => { - port.postMessage(value, transfer) + const callback = (args, transfer = []) => { + port.postMessage(args, transfer) } return callback diff --git a/packages/ipfs-message-port-protocol/test/core.browser.js b/packages/ipfs-message-port-protocol/test/core.browser.js index ae2ea46b56..f22e60f5c1 100644 --- a/packages/ipfs-message-port-protocol/test/core.browser.js +++ b/packages/ipfs-message-port-protocol/test/core.browser.js @@ -32,10 +32,10 @@ describe('core', function () { await move(encodeCallback(callback, transfer), transfer) ) - remote(54) + remote([54]) expect(await receive()).to.be.equal(54) - remote({ hello: 'world' }) + remote([{ hello: 'world' }]) expect(await receive()).to.be.deep.equal({ hello: 'world' }) }) @@ -55,11 +55,11 @@ describe('core', function () { await move(encodeCallback(callback, transfer), transfer) ) - remote({ hello: uint8ArrayFromString('world') }) + remote([{ hello: uint8ArrayFromString('world') }]) expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') }) const world = uint8ArrayFromString('world') - remote({ hello: world }, [world.buffer]) + remote([{ hello: world }], [world.buffer]) expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') }) expect(world.buffer).property('byteLength', 0, 'buffer was cleared') diff --git a/packages/ipfs-message-port-server/src/core.js b/packages/ipfs-message-port-server/src/core.js index 97092e2187..dea2eb81a1 100644 --- a/packages/ipfs-message-port-server/src/core.js +++ b/packages/ipfs-message-port-server/src/core.js @@ -25,8 +25,7 @@ const { decodeCID, encodeCID } = require('ipfs-message-port-protocol/src/cid') */ /** - * @template T - * @typedef {import('ipfs-message-port-protocol/src/core').RemoteCallback} RemoteCallback + * @typedef {import('ipfs-message-port-protocol/src/core').RemoteCallback} RemoteCallback */ /** @@ -42,7 +41,7 @@ const { decodeCID, encodeCID } = require('ipfs-message-port-protocol/src/cid') * @property {HashAlg} [hashAlg] * @property {boolean} [onlyHash] * @property {boolean} [pin] - * @property {RemoteCallback|void} [progress] + * @property {RemoteCallback|void} [progress] * @property {boolean} [rawLeaves] * @property {number} [shardSplitThreshold] * @property {boolean} [trickle] @@ -130,6 +129,13 @@ exports.CoreService = class CoreService { signal } = query + let progressCallback + + if (progress) { + const fn = decodeCallback(progress) + progressCallback = (bytes, fileName) => fn([bytes, fileName]) + } + const options = { chunker, cidVersion, @@ -142,7 +148,7 @@ exports.CoreService = class CoreService { trickle, wrapWithDirectory, timeout, - progress: progress != null ? decodeCallback(progress) : undefined, + progress: progressCallback, signal } @@ -176,6 +182,13 @@ exports.CoreService = class CoreService { signal } = query + let progressCallback + + if (progress) { + const fn = decodeCallback(progress) + progressCallback = (bytes, fileName) => fn([bytes, fileName]) + } + const options = { chunker, cidVersion, @@ -188,7 +201,7 @@ exports.CoreService = class CoreService { trickle, wrapWithDirectory, timeout, - progress: progress != null ? decodeCallback(progress) : undefined, + progress: progressCallback, signal }