From 908f92870fe772f871a9fea575bac1749c3e141c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 27 Sep 2021 17:12:41 +0100 Subject: [PATCH 1/5] fix: do not accept single items for ipfs.add The types allow passing single items to `ipfs.addAll` and multiple items to `ipfs.add`. Instead, only accept single items to `ipfs.add` and streams of item to `ipfs.addAll` and fail with a more helpful error message if you do not do this. --- packages/interface-ipfs-core/src/add-all.js | 20 ++ packages/interface-ipfs-core/src/add.js | 14 + packages/ipfs-cli/src/parser.js | 1 - packages/ipfs-core-utils/package.json | 14 +- .../src/files/normalise-candidate-multiple.js | 114 +++++++++ ...alise.js => normalise-candidate-single.js} | 34 +-- .../src/files/normalise-content.browser.js | 2 +- .../src/files/normalise-content.js | 29 +-- ...js => normalise-input-multiple.browser.js} | 11 +- .../src/files/normalise-input-multiple.js | 21 ++ .../files/normalise-input-single.browser.js | 24 ++ ...ise-input.js => normalise-input-single.js} | 7 +- .../src/multipart-request.browser.js | 5 +- .../ipfs-core-utils/src/multipart-request.js | 3 +- .../src/multipart-request.node.js | 7 +- .../files/normalise-input-multiple.spec.js | 240 ++++++++++++++++++ ...spec.js => normalise-input-single.spec.js} | 4 +- .../ipfs-core/src/components/add-all/index.js | 2 +- packages/ipfs-core/src/components/add.js | 3 +- .../ipfs-grpc-client/src/core-api/add-all.js | 2 +- packages/ipfs-http-client/src/add.js | 3 +- packages/ipfs-http-client/src/block/put.js | 2 +- .../ipfs-http-client/src/config/replace.js | 2 +- packages/ipfs-http-client/src/dag/put.js | 2 +- packages/ipfs-http-client/src/dht/put.js | 2 +- packages/ipfs-http-client/src/files/write.js | 4 +- .../src/object/patch/append-data.js | 2 +- .../src/object/patch/set-data.js | 2 +- .../ipfs-http-client/src/pubsub/publish.js | 2 +- 29 files changed, 496 insertions(+), 82 deletions(-) create mode 100644 packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js rename packages/ipfs-core-utils/src/files/{normalise.js => normalise-candidate-single.js} (70%) rename packages/ipfs-core-utils/src/files/{normalise-input.browser.js => normalise-input-multiple.browser.js} (61%) create mode 100644 packages/ipfs-core-utils/src/files/normalise-input-multiple.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input-single.browser.js rename packages/ipfs-core-utils/src/files/{normalise-input.js => normalise-input-single.js} (65%) create mode 100644 packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js rename packages/ipfs-core-utils/test/files/{normalise-input.spec.js => normalise-input-single.spec.js} (98%) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 09035d2863..62101f303d 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -301,6 +301,26 @@ export function testAddAll (factory, options) { await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejected() }) + it('should fail when passed single file objects', async () => { + const nonValid = { content: 'hello world' } + + // @ts-expect-error nonValid is non valid + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + + it('should fail when passed single strings', async () => { + const nonValid = 'hello world' + + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + + it('should fail when passed single buffers', async () => { + const nonValid = uint8ArrayFromString('hello world') + + // @ts-expect-error nonValid is non valid + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index aa03c4bb2a..104e85075e 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -244,6 +244,20 @@ export function testAdd (factory, options) { await expect(ipfs.add(null)).to.eventually.be.rejected() }) + it('should fail when passed multiple file objects', async () => { + const nonValid = [{ content: 'hello' }, { content: 'world' }] + + // @ts-expect-error nonValid is non valid + await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) + }) + + it('should fail when passed multiple strings', async () => { + const nonValid = ['hello', 'world'] + + // @ts-expect-error nonValid is non valid + await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) + }) + it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } diff --git a/packages/ipfs-cli/src/parser.js b/packages/ipfs-cli/src/parser.js index 274a2944af..df08a9b604 100644 --- a/packages/ipfs-cli/src/parser.js +++ b/packages/ipfs-cli/src/parser.js @@ -1,4 +1,3 @@ - import yargs from 'yargs' import { ipfsPathHelp, disablePrinting } from './utils.js' import { commandList } from './commands/index.js' diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 07b71fe23b..a559120728 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -38,11 +38,17 @@ ".": { "import": "./src/index.js" }, - "./files/normalise-input": { - "import": "./src/files/normalise-input.js" + "./files/normalise-input-single": { + "import": "./src/files/normalise-input-single.js" }, - "./files/normalise-input.browser": { - "import": "./src/files/normalise-input.browser.js" + "./files/normalise-input-single.browser": { + "import": "./src/files/normalise-input-single.browser.js" + }, + "./files/normalise-input-multiple": { + "import": "./src/files/normalise-input-multiple.js" + }, + "./files/normalise-input-multiple.browser": { + "import": "./src/files/normalise-input-multiple.browser.js" }, "./files/normalise-content": { "import": "./src/files/normalise-content.js" diff --git a/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js new file mode 100644 index 0000000000..1e254bfd2a --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js @@ -0,0 +1,114 @@ +import errCode from 'err-code' +import browserStreamToIt from 'browser-readablestream-to-it' +import itPeekable from 'it-peekable' +import map from 'it-map' +import { + isBytes, + isBlob, + isReadableStream, + isFileObject +} from './utils.js' +import { + parseMtime, + parseMode +} from 'ipfs-unixfs' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate + * @typedef {import('ipfs-core-types/src/utils').ToContent} ToContent + * @typedef {import('ipfs-unixfs-importer').ImportCandidate} ImporterImportCandidate + * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream + */ + +/** + * @param {ImportCandidateStream} input + * @param {(content:ToContent) => Promise>} normaliseContent + */ +// eslint-disable-next-line complexity +export async function * normaliseCandidateMultiple (input, normaliseContent) { + // String + // Uint8Array|ArrayBuffer|TypedArray + // Blob|File + if (typeof input === 'string' || input instanceof String || isBytes(input) || isBlob(input)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + // Browser ReadableStream + if (isReadableStream(input)) { + input = browserStreamToIt(input) + } + + // Iterable + if (Symbol.iterator in input || Symbol.asyncIterator in input) { + /** @type {any} */ + // @ts-ignore it's (async)interable + const peekable = itPeekable(input) + + /** @type {any} value **/ + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * [] + return + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + // (Async)Iterable + if (value._readableState) { + // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content + yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) + return + } + + // (Async)Iterable<(Async)Iterable> + // (Async)Iterable> + // ReadableStream<(Async)Iterable> + // ReadableStream> + if (isFileObject(value) || value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) { + yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) + return + } + } + + // { path, content: ? } + // Note: Detected _after_ (Async)Iterable because Node.js fs.ReadStreams have a + // `path` property that passes this check. + if (isFileObject(input)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +/** + * @param {ImportCandidate} input + * @param {(content:ToContent) => Promise>} normaliseContent + */ +async function toFileObject (input, normaliseContent) { + // @ts-ignore - Those properties don't exist on most input types + const { path, mode, mtime, content } = input + + /** @type {ImporterImportCandidate} */ + const file = { + path: path || '', + mode: parseMode(mode), + mtime: parseMtime(mtime) + } + + if (content) { + file.content = await normaliseContent(content) + } else if (!path) { // Not already a file object with path or content prop + // @ts-ignore - input still can be different ToContent + file.content = await normaliseContent(input) + } + + return file +} diff --git a/packages/ipfs-core-utils/src/files/normalise.js b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js similarity index 70% rename from packages/ipfs-core-utils/src/files/normalise.js rename to packages/ipfs-core-utils/src/files/normalise-candidate-single.js index 19b888943d..760a1df226 100644 --- a/packages/ipfs-core-utils/src/files/normalise.js +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js @@ -21,11 +21,11 @@ import { */ /** - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidate} input * @param {(content:ToContent) => Promise>} normaliseContent */ // eslint-disable-next-line complexity -export async function * normalise (input, normaliseContent) { +export async function * normaliseCandidateSingle (input, normaliseContent) { if (input === null || input === undefined) { throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') } @@ -50,8 +50,7 @@ export async function * normalise (input, normaliseContent) { // Iterable if (Symbol.iterator in input || Symbol.asyncIterator in input) { - /** @type {any} */ - // @ts-ignore it's (async)interable + // @ts-ignore it's (async)iterable const peekable = itPeekable(input) /** @type {any} value **/ @@ -59,7 +58,7 @@ export async function * normalise (input, normaliseContent) { if (done) { // make sure empty iterators result in empty files - yield * [] + yield { content: [] } return } @@ -72,40 +71,25 @@ export async function * normalise (input, normaliseContent) { return } - // fs.ReadStream + // (Async)Iterable if (value._readableState) { - // @ts-ignore Node readable streams have a `.path` property so we need to pass it as the content + // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) return } - // (Async)Iterable - // (Async)Iterable - // (Async)Iterable<{ path, content }> - if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) { - yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) - return - } - - // (Async)Iterable<(Async)Iterable> - // (Async)Iterable> - // ReadableStream<(Async)Iterable> - // ReadableStream> - if (value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) { - yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) - return - } + throw errCode(new Error('Unexpected input: multiple items passed'), 'ERR_UNEXPECTED_INPUT') } // { path, content: ? } - // Note: Detected _after_ (Async)Iterable because Node.js streams have a + // Note: Detected _after_ (Async)Iterable because Node.js fs.ReadStreams have a // `path` property that passes this check. if (isFileObject(input)) { yield toFileObject(input, normaliseContent) return } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + throw errCode(new Error('Unexpected input: cannot convert "' + typeof input + '" into ImportCandidate'), 'ERR_UNEXPECTED_INPUT') } /** diff --git a/packages/ipfs-core-utils/src/files/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-content.browser.js index b208eba8c9..09508f0700 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-content.browser.js @@ -9,7 +9,7 @@ import { } from './utils.js' /** - * @param {import('./normalise').ToContent} input + * @param {import('ipfs-core-types/src/utils').ToContent} input */ export async function normaliseContent (input) { // Bytes diff --git a/packages/ipfs-core-utils/src/files/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-content.js index 4d3e4fa9d6..0cdfa0ab70 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-content.js @@ -12,31 +12,29 @@ import { } from './utils.js' /** - * @param {import('./normalise').ToContent} input + * @template T + * @param {T} thing */ -export async function normaliseContent (input) { - return toAsyncGenerator(input) +async function * toAsyncIterable (thing) { + yield thing } /** - * @param {import('./normalise').ToContent} input + * @param {import('ipfs-core-types/src/utils').ToContent} input */ -async function * toAsyncGenerator (input) { +export async function normaliseContent (input) { // Bytes | String if (isBytes(input)) { - yield toBytes(input) - return + return toAsyncIterable(toBytes(input)) } if (typeof input === 'string' || input instanceof String) { - yield toBytes(input.toString()) - return + return toAsyncIterable(toBytes(input.toString())) } // Blob if (isBlob(input)) { - yield * blobToIt(input) - return + return blobToIt(input) } // Browser stream @@ -54,22 +52,19 @@ async function * toAsyncGenerator (input) { if (done) { // make sure empty iterators result in empty files - yield * [] - return + return toAsyncIterable(new Uint8Array(0)) } peekable.push(value) // (Async)Iterable if (Number.isInteger(value)) { - yield Uint8Array.from((await all(peekable))) - return + return toAsyncIterable(Uint8Array.from(await all(peekable))) } // (Async)Iterable if (isBytes(value) || typeof value === 'string' || value instanceof String) { - yield * map(peekable, toBytes) - return + return map(peekable, toBytes) } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input.browser.js b/packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js similarity index 61% rename from packages/ipfs-core-utils/src/files/normalise-input.browser.js rename to packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js index 549d4473e3..a8d4595150 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js @@ -1,14 +1,13 @@ import { normaliseContent } from './normalise-content.browser.js' -import { normalise } from './normalise.js' +import { normaliseCandidateMultiple } from './normalise-candidate-multiple.js' /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate * @typedef {import('ipfs-core-types/src/utils').BrowserImportCandidate} BrowserImportCandidate */ /** - * Transforms any of the `ipfs.add` input types into + * Transforms any of the `ipfs.addAll` input types into * * ``` * AsyncIterable<{ path, mode, mtime, content: Blob }> @@ -16,10 +15,10 @@ import { normalise } from './normalise.js' * * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options * - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidateStream} input * @returns {AsyncGenerator} */ export function normaliseInput (input) { - // @ts-ignore normaliseContent returns Blob and not AsyncIterator - return normalise(input, normaliseContent) + // @ts-expect-error browser normaliseContent returns a Blob not an AsyncIterable + return normaliseCandidateMultiple(input, normaliseContent, true) } diff --git a/packages/ipfs-core-utils/src/files/normalise-input-multiple.js b/packages/ipfs-core-utils/src/files/normalise-input-multiple.js new file mode 100644 index 0000000000..b95760edc5 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input-multiple.js @@ -0,0 +1,21 @@ +import { normaliseContent } from './normalise-content.js' +import { normaliseCandidateMultiple } from './normalise-candidate-multiple.js' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream + */ + +/** + * Transforms any of the `ipfs.addAll` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param {ImportCandidateStream} input + */ +export function normaliseInput (input) { + return normaliseCandidateMultiple(input, normaliseContent) +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js b/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js new file mode 100644 index 0000000000..50af999fc1 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js @@ -0,0 +1,24 @@ +import { normaliseContent } from './normalise-content.browser.js' +import { normaliseCandidateSingle } from './normalise-candidate-single.js' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate + * @typedef {import('ipfs-core-types/src/utils').BrowserImportCandidate} BrowserImportCandidate + */ + +/** + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: Blob }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param {ImportCandidate} input + * @returns {BrowserImportCandidate} + */ +export function normaliseInput (input) { + // @ts-expect-error browser normaliseContent returns a Blob not an AsyncIterable + return normaliseCandidateSingle(input, normaliseContent) +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input-single.js similarity index 65% rename from packages/ipfs-core-utils/src/files/normalise-input.js rename to packages/ipfs-core-utils/src/files/normalise-input-single.js index c3bba3d1e3..0946d1e77c 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input-single.js @@ -1,8 +1,7 @@ import { normaliseContent } from './normalise-content.js' -import { normalise } from './normalise.js' +import { normaliseCandidateSingle } from './normalise-candidate-single.js' /** - * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ @@ -15,8 +14,8 @@ import { normalise } from './normalise.js' * * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options * - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidate} input */ export function normaliseInput (input) { - return normalise(input, normaliseContent) + return normaliseCandidateSingle(input, normaliseContent) } diff --git a/packages/ipfs-core-utils/src/multipart-request.browser.js b/packages/ipfs-core-utils/src/multipart-request.browser.js index 2f231b3352..be444082c7 100644 --- a/packages/ipfs-core-utils/src/multipart-request.browser.js +++ b/packages/ipfs-core-utils/src/multipart-request.browser.js @@ -1,16 +1,15 @@ // Import browser version otherwise electron-renderer will end up with node // version and fail. -import { normaliseInput } from './files/normalise-input.browser.js' +import { normaliseInput } from './files/normalise-input-multiple.browser.js' import { modeToString } from './mode-to-string.js' /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] */ diff --git a/packages/ipfs-core-utils/src/multipart-request.js b/packages/ipfs-core-utils/src/multipart-request.js index e46fdd4780..2e2a503e23 100644 --- a/packages/ipfs-core-utils/src/multipart-request.js +++ b/packages/ipfs-core-utils/src/multipart-request.js @@ -4,12 +4,11 @@ import { multipartRequest as multipartRequestBrowser } from './multipart-request import { nanoid } from 'nanoid' /** - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] * @param {string} [boundary] diff --git a/packages/ipfs-core-utils/src/multipart-request.node.js b/packages/ipfs-core-utils/src/multipart-request.node.js index fda8a74246..d406deb50f 100644 --- a/packages/ipfs-core-utils/src/multipart-request.node.js +++ b/packages/ipfs-core-utils/src/multipart-request.node.js @@ -1,4 +1,4 @@ -import { normaliseInput } from './files/normalise-input.js' +import { normaliseInput } from './files/normalise-input-multiple.js' import { nanoid } from 'nanoid' import { modeToString } from './mode-to-string.js' import mergeOpts from 'merge-options' @@ -11,18 +11,17 @@ const log = debug('ipfs:core-utils:multipart-request') /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] * @param {string} [boundary] */ export async function multipartRequest (source, abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source */ async function * streamFiles (source) { try { diff --git a/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js new file mode 100644 index 0000000000..8f49a833f5 --- /dev/null +++ b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js @@ -0,0 +1,240 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/utils/chai.js' +import blobToIt from 'blob-to-it' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import all from 'it-all' +import { File } from '@web-std/file' +import { normaliseInput } from '../../src/files/normalise-input-multiple.js' +import { isNode } from 'ipfs-utils/src/env.js' +import resolve from 'aegir/utils/resolve.js' + +const { Blob, ReadableStream } = globalThis + +const STRING = () => 'hello world' +const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers +const BUFFER = () => uint8ArrayFromString(STRING()) +const ARRAY = () => Array.from(BUFFER()) +const TYPEDARRAY = () => Uint8Array.from(ARRAY()) +/** @type {() => Blob} */ +let BLOB + +if (Blob) { + BLOB = () => new Blob([ + STRING() + ]) +} + +/** + * @param {import('ipfs-unixfs-importer').ImportCandidate[]} input + */ +async function verifyNormalisation (input) { + expect(input.length).to.equal(1) + expect(input[0].path).to.equal('') + + let content = input[0].content + + if (Blob && content instanceof Blob) { + content = blobToIt(content) + } + + if (!content || content instanceof Uint8Array) { + throw new Error('Content expected') + } + + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) +} + +/** + * @param {*} input + */ +async function testContent (input) { + const result = await all(normaliseInput(input)) + + await verifyNormalisation(result) +} + +/** + * @template T + * @param {T} thing + * @returns {T[]} + */ +function iterableOf (thing) { + return [thing] +} + +/** + * @template T + * @param {T} thing + * @returns {AsyncIterable} + */ +function asyncIterableOf (thing) { + return (async function * () { // eslint-disable-line require-await + yield thing + }()) +} + +/** + * @param {*} thing + */ +function browserReadableStreamOf (thing) { + return new ReadableStream({ + start (controller) { + controller.enqueue(thing) + controller.close() + } + }) +} + +describe('normalise-input-multiple', function () { + /** + * @param {() => any} content + * @param {string} name + * @param {boolean} isBytes + */ + function testInputType (content, name, isBytes) { + it(name, async function () { + await testContent(content()) + }) + + if (isBytes) { + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + + it(`Iterable<${name}>`, async function () { + await testContent(iterableOf(content())) + }) + + it(`AsyncIterable<${name}>`, async function () { + await testContent(asyncIterableOf(content())) + }) + } + + it(`{ path: '', content: ${name} }`, async function () { + await testContent({ path: '', content: content() }) + }) + + if (isBytes) { + if (ReadableStream) { + it(`{ path: '', content: ReadableStream<${name}> }`, async function () { + await testContent({ path: '', content: browserReadableStreamOf(content()) }) + }) + } + + it(`{ path: '', content: Iterable<${name}> }`, async function () { + await testContent({ path: '', content: iterableOf(content()) }) + }) + + it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { + await testContent({ path: '', content: asyncIterableOf(content()) }) + }) + } + + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + + it(`Iterable<{ path: '', content: ${name} }`, async function () { + await testContent(iterableOf({ path: '', content: content() })) + }) + + it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { + await testContent(asyncIterableOf({ path: '', content: content() })) + }) + + if (isBytes) { + if (ReadableStream) { + it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: iterableOf(content()) })) + }) + + it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) + }) + + if (ReadableStream) { + it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) + }) + + it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content()) })) + }) + } + } + + describe('String', () => { + testInputType(STRING, 'String', true) + testInputType(NEWSTRING, 'new String()', true) + }) + + describe('Buffer', () => { + testInputType(BUFFER, 'Buffer', true) + }) + + describe('Blob', () => { + if (!Blob) { + return + } + + testInputType(BLOB, 'Blob', false) + }) + + describe('@web-std/file', () => { + it('normalizes File input', async () => { + const FILE = new File([BUFFER()], 'test-file.txt') + + await testContent(FILE) + }) + }) + + describe('Iterable', () => { + testInputType(ARRAY, 'Iterable', false) + }) + + describe('TypedArray', () => { + testInputType(TYPEDARRAY, 'TypedArray', true) + }) + + if (isNode) { + /** @type {import('fs')} */ + let fs + + before(async () => { + fs = await import('fs') + }) + + describe('Node fs.ReadStream', () => { + const NODEFSREADSTREAM = () => { + const path = resolve('test/fixtures/file.txt', 'ipfs-core-utils') + + return fs.createReadStream(path) + } + + testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) + + it('Iterable', async function () { + await testContent(iterableOf(NODEFSREADSTREAM())) + }) + + it('AsyncIterable', async function () { + await testContent(asyncIterableOf(NODEFSREADSTREAM())) + }) + }) + } +}) diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js similarity index 98% rename from packages/ipfs-core-utils/test/files/normalise-input.spec.js rename to packages/ipfs-core-utils/test/files/normalise-input-single.spec.js index 7f1c2d8706..f44ba17012 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js @@ -5,7 +5,7 @@ import blobToIt from 'blob-to-it' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import all from 'it-all' import { File } from '@web-std/file' -import { normaliseInput } from '../../src/files/normalise-input.js' +import { normaliseInput } from '../../src/files/normalise-input-single.js' import { isNode } from 'ipfs-utils/src/env.js' import resolve from 'aegir/utils/resolve.js' @@ -86,7 +86,7 @@ function browserReadableStreamOf (thing) { }) } -describe('normalise-input', function () { +describe('normalise-input-single', function () { /** * @param {() => any} content * @param {string} name diff --git a/packages/ipfs-core/src/components/add-all/index.js b/packages/ipfs-core/src/components/add-all/index.js index a0f765d2dc..f69c6f54e9 100644 --- a/packages/ipfs-core/src/components/add-all/index.js +++ b/packages/ipfs-core/src/components/add-all/index.js @@ -1,5 +1,5 @@ import { importer } from 'ipfs-unixfs-importer' -import { normaliseInput } from 'ipfs-core-utils/files/normalise-input' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-multiple' import { parseChunkerString } from './utils.js' import { pipe } from 'it-pipe' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' diff --git a/packages/ipfs-core/src/components/add.js b/packages/ipfs-core/src/components/add.js index 2a2c500eab..b0cd6e9cb8 100644 --- a/packages/ipfs-core/src/components/add.js +++ b/packages/ipfs-core/src/components/add.js @@ -1,4 +1,5 @@ import last from 'it-last' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-single' /** * @param {Object} context @@ -10,7 +11,7 @@ export function createAdd ({ addAll }) { */ async function add (entry, options = {}) { // @ts-ignore TODO: https://github.com/ipfs/js-ipfs/issues/3290 - const result = await last(addAll(entry, options)) + const result = await last(addAll(normaliseInput(entry), options)) // Note this should never happen as `addAll` should yield at least one item // but to satisfy type checker we perfom this check and for good measure // throw an error in case it does happen. diff --git a/packages/ipfs-grpc-client/src/core-api/add-all.js b/packages/ipfs-grpc-client/src/core-api/add-all.js index 690d731f2a..5304123f53 100644 --- a/packages/ipfs-grpc-client/src/core-api/add-all.js +++ b/packages/ipfs-grpc-client/src/core-api/add-all.js @@ -1,4 +1,4 @@ -import { normaliseInput } from 'ipfs-core-utils/files/normalise-input' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-multiple' import { CID } from 'multiformats/cid' import { bidiToDuplex } from '../utils/bidi-to-duplex.js' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' diff --git a/packages/ipfs-http-client/src/add.js b/packages/ipfs-http-client/src/add.js index d668440b7c..979fc9a35d 100644 --- a/packages/ipfs-http-client/src/add.js +++ b/packages/ipfs-http-client/src/add.js @@ -1,6 +1,7 @@ import { createAddAll } from './add-all.js' import last from 'it-last' import { configure } from './lib/configure.js' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-single' /** * @typedef {import('./types').HTTPClientExtraOptions} HTTPClientExtraOptions @@ -18,7 +19,7 @@ export function createAdd (options) { */ async function add (input, options = {}) { // @ts-ignore - last may return undefined if source is empty - return await last(all(input, options)) + return await last(all(normaliseInput(input), options)) } return add })(options) diff --git a/packages/ipfs-http-client/src/block/put.js b/packages/ipfs-http-client/src/block/put.js index 36ad65d1ae..0b1480402d 100644 --- a/packages/ipfs-http-client/src/block/put.js +++ b/packages/ipfs-http-client/src/block/put.js @@ -25,7 +25,7 @@ export const createPut = configure(api => { signal: signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) res = await response.json() diff --git a/packages/ipfs-http-client/src/config/replace.js b/packages/ipfs-http-client/src/config/replace.js index 196288a443..47e9268e54 100644 --- a/packages/ipfs-http-client/src/config/replace.js +++ b/packages/ipfs-http-client/src/config/replace.js @@ -23,7 +23,7 @@ export const createReplace = configure(api => { signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(uint8ArrayFromString(JSON.stringify(config)), controller, options.headers) + await multipartRequest([uint8ArrayFromString(JSON.stringify(config))], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index ded70c968d..6192e1750a 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -39,7 +39,7 @@ export const createPut = (codecs, options) => { signal, searchParams: toUrlSearchParams(settings), ...( - await multipartRequest(serialized, controller, settings.headers) + await multipartRequest([serialized], controller, settings.headers) ) }) const data = await res.json() diff --git a/packages/ipfs-http-client/src/dht/put.js b/packages/ipfs-http-client/src/dht/put.js index 536e1099d1..259a70884a 100644 --- a/packages/ipfs-http-client/src/dht/put.js +++ b/packages/ipfs-http-client/src/dht/put.js @@ -28,7 +28,7 @@ export const createPut = configure(api => { ...options }), ...( - await multipartRequest(value, controller, options.headers) + await multipartRequest([value], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/files/write.js b/packages/ipfs-http-client/src/files/write.js index 8fd918589f..13bf3ac3f9 100644 --- a/packages/ipfs-http-client/src/files/write.js +++ b/packages/ipfs-http-client/src/files/write.js @@ -29,12 +29,12 @@ export const createWrite = configure(api => { ...options }), ...( - await multipartRequest({ + await multipartRequest([{ content: input, path: 'arg', mode: modeToString(options.mode), mtime: parseMtime(options.mtime) - }, controller, options.headers) + }], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/object/patch/append-data.js b/packages/ipfs-http-client/src/object/patch/append-data.js index 2e4f17bd4a..1496b15682 100644 --- a/packages/ipfs-http-client/src/object/patch/append-data.js +++ b/packages/ipfs-http-client/src/object/patch/append-data.js @@ -26,7 +26,7 @@ export const createAppendData = configure(api => { ...options }), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/object/patch/set-data.js b/packages/ipfs-http-client/src/object/patch/set-data.js index fcc9037711..cb9663f728 100644 --- a/packages/ipfs-http-client/src/object/patch/set-data.js +++ b/packages/ipfs-http-client/src/object/patch/set-data.js @@ -28,7 +28,7 @@ export const createSetData = configure(api => { ...options }), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/pubsub/publish.js b/packages/ipfs-http-client/src/pubsub/publish.js index 53191f4c69..5c51dca64e 100644 --- a/packages/ipfs-http-client/src/pubsub/publish.js +++ b/packages/ipfs-http-client/src/pubsub/publish.js @@ -27,7 +27,7 @@ export const createPublish = configure(api => { signal, searchParams, ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) From 4c2941b4b9f5d464dbd3a262ab121bac4a085ae1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 28 Sep 2021 17:02:07 +0100 Subject: [PATCH 2/5] chore: fix tests for add input --- .../src/files/normalise-candidate-multiple.js | 19 ++- .../src/files/normalise-candidate-single.js | 13 +- .../files/normalise-input-multiple.spec.js | 129 ++++++++++++------ .../test/files/normalise-input-single.spec.js | 117 ++++++++++------ packages/ipfs-core-utils/test/tests.spec.js | 3 +- 5 files changed, 178 insertions(+), 103 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js index 1e254bfd2a..4ba85ee23d 100644 --- a/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js @@ -29,8 +29,10 @@ export async function * normaliseCandidateMultiple (input, normaliseContent) { // String // Uint8Array|ArrayBuffer|TypedArray // Blob|File - if (typeof input === 'string' || input instanceof String || isBytes(input) || isBlob(input)) { - throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + // fs.ReadStream + // @ts-expect-error _readableState is a property of a node fs.ReadStream + if (typeof input === 'string' || input instanceof String || isBytes(input) || isBlob(input) || input._readableState) { + throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT') } // Browser ReadableStream @@ -57,8 +59,8 @@ export async function * normaliseCandidateMultiple (input, normaliseContent) { // (Async)Iterable // (Async)Iterable - if (Number.isInteger(value) || isBytes(value)) { - throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + if (Number.isInteger(value)) { + throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT') } // (Async)Iterable @@ -68,11 +70,16 @@ export async function * normaliseCandidateMultiple (input, normaliseContent) { return } + if (isBytes(value)) { + yield toFileObject({ content: peekable }, normaliseContent) + return + } + // (Async)Iterable<(Async)Iterable> // (Async)Iterable> // ReadableStream<(Async)Iterable> // ReadableStream> - if (isFileObject(value) || value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) { + if (isFileObject(value) || value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value) || isBlob(value)) { yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) return } @@ -82,7 +89,7 @@ export async function * normaliseCandidateMultiple (input, normaliseContent) { // Note: Detected _after_ (Async)Iterable because Node.js fs.ReadStreams have a // `path` property that passes this check. if (isFileObject(input)) { - throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT') } throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') diff --git a/packages/ipfs-core-utils/src/files/normalise-candidate-single.js b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js index 760a1df226..f66afefe00 100644 --- a/packages/ipfs-core-utils/src/files/normalise-candidate-single.js +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js @@ -1,7 +1,6 @@ import errCode from 'err-code' import browserStreamToIt from 'browser-readablestream-to-it' import itPeekable from 'it-peekable' -import map from 'it-map' import { isBytes, isBlob, @@ -66,19 +65,13 @@ export async function * normaliseCandidateSingle (input, normaliseContent) { // (Async)Iterable // (Async)Iterable - if (Number.isInteger(value) || isBytes(value)) { + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value) || typeof value === 'string' || value instanceof String) { yield toFileObject(peekable, normaliseContent) return } - // (Async)Iterable - if (value._readableState) { - // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content - yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) - return - } - - throw errCode(new Error('Unexpected input: multiple items passed'), 'ERR_UNEXPECTED_INPUT') + throw errCode(new Error('Unexpected input: multiple items passed - if you are using ipfs.add, please use ipfs.addAll instead'), 'ERR_UNEXPECTED_INPUT') } // { path, content: ? } diff --git a/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js index 8f49a833f5..21614201e4 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js @@ -16,6 +16,7 @@ const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new- const BUFFER = () => uint8ArrayFromString(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) +const FILE = () => new File([BUFFER()], 'test-file.txt') /** @type {() => Blob} */ let BLOB @@ -54,6 +55,14 @@ async function testContent (input) { await verifyNormalisation(result) } +/** + * @param {*} input + * @param {RegExp} message + */ +async function testFailure (input, message) { + await expect(all(normaliseInput(input))).to.eventually.be.rejectedWith(message) +} + /** * @template T * @param {T} thing @@ -90,14 +99,14 @@ describe('normalise-input-multiple', function () { /** * @param {() => any} content * @param {string} name - * @param {boolean} isBytes + * @param {{ acceptStream: boolean, acceptContentStream: boolean }} options */ - function testInputType (content, name, isBytes) { - it(name, async function () { - await testContent(content()) + function testInputType (content, name, { acceptStream, acceptContentStream }) { + it(`Failure ${name}`, async function () { + await testFailure(content(), /single item passed/) }) - if (isBytes) { + if (acceptStream) { if (ReadableStream) { it(`ReadableStream<${name}>`, async function () { await testContent(browserReadableStreamOf(content())) @@ -111,43 +120,35 @@ describe('normalise-input-multiple', function () { it(`AsyncIterable<${name}>`, async function () { await testContent(asyncIterableOf(content())) }) - } - - it(`{ path: '', content: ${name} }`, async function () { - await testContent({ path: '', content: content() }) - }) - - if (isBytes) { + } else { if (ReadableStream) { - it(`{ path: '', content: ReadableStream<${name}> }`, async function () { - await testContent({ path: '', content: browserReadableStreamOf(content()) }) + it(`Failure ReadableStream<${name}>`, async function () { + await testFailure(browserReadableStreamOf(content()), /single item passed/) }) } - it(`{ path: '', content: Iterable<${name}> }`, async function () { - await testContent({ path: '', content: iterableOf(content()) }) + it(`Failure Iterable<${name}>`, async function () { + await testFailure(iterableOf(content()), /single item passed/) }) - it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { - await testContent({ path: '', content: asyncIterableOf(content()) }) + it(`Failure AsyncIterable<${name}>`, async function () { + await testFailure(asyncIterableOf(content()), /single item passed/) }) } - if (ReadableStream) { - it(`ReadableStream<${name}>`, async function () { - await testContent(browserReadableStreamOf(content())) - }) - } + it(`Failure { path: '', content: ${name} }`, async function () { + await testFailure({ path: '', content: content() }, /single item passed/) + }) - it(`Iterable<{ path: '', content: ${name} }`, async function () { + it(`Iterable<{ path: '', content: ${name} }>`, async function () { await testContent(iterableOf({ path: '', content: content() })) }) - it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { + it(`AsyncIterable<{ path: '', content: ${name} }>`, async function () { await testContent(asyncIterableOf({ path: '', content: content() })) }) - if (isBytes) { + if (acceptContentStream) { if (ReadableStream) { it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) @@ -175,16 +176,53 @@ describe('normalise-input-multiple', function () { it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content()) })) }) + } else { + if (ReadableStream) { + it(`Failure Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: browserReadableStreamOf(content()) }), /Unexpected input/) + }) + } + + it(`Failure Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: iterableOf(content()) }), /Unexpected input/) + }) + + it(`Failure Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: asyncIterableOf(content()) }), /Unexpected input/) + }) + + if (ReadableStream) { + it(`Failure AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) }), /Unexpected input/) + }) + } + + it(`Failure AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: iterableOf(content()) }), /Unexpected input/) + }) + + it(`Failure AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: asyncIterableOf(content()) }), /Unexpected input/) + }) } } describe('String', () => { - testInputType(STRING, 'String', true) - testInputType(NEWSTRING, 'new String()', true) + testInputType(STRING, 'String', { + acceptStream: true, + acceptContentStream: true + }) + testInputType(NEWSTRING, 'new String()', { + acceptStream: true, + acceptContentStream: true + }) }) describe('Buffer', () => { - testInputType(BUFFER, 'Buffer', true) + testInputType(BUFFER, 'Buffer', { + acceptStream: true, + acceptContentStream: true + }) }) describe('Blob', () => { @@ -192,23 +230,31 @@ describe('normalise-input-multiple', function () { return } - testInputType(BLOB, 'Blob', false) + testInputType(BLOB, 'Blob', { + acceptStream: true, + acceptContentStream: false + }) }) describe('@web-std/file', () => { - it('normalizes File input', async () => { - const FILE = new File([BUFFER()], 'test-file.txt') - - await testContent(FILE) + testInputType(FILE, 'File', { + acceptStream: true, + acceptContentStream: false }) }) describe('Iterable', () => { - testInputType(ARRAY, 'Iterable', false) + testInputType(ARRAY, 'Iterable', { + acceptStream: true, + acceptContentStream: false + }) }) describe('TypedArray', () => { - testInputType(TYPEDARRAY, 'TypedArray', true) + testInputType(TYPEDARRAY, 'TypedArray', { + acceptStream: true, + acceptContentStream: true + }) }) if (isNode) { @@ -226,14 +272,9 @@ describe('normalise-input-multiple', function () { return fs.createReadStream(path) } - testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) - - it('Iterable', async function () { - await testContent(iterableOf(NODEFSREADSTREAM())) - }) - - it('AsyncIterable', async function () { - await testContent(asyncIterableOf(NODEFSREADSTREAM())) + testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', { + acceptStream: true, + acceptContentStream: false }) }) } diff --git a/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js index f44ba17012..6113e1db67 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js @@ -16,6 +16,7 @@ const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new- const BUFFER = () => uint8ArrayFromString(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) +const FILE = () => new File([BUFFER()], 'test-file.txt') /** @type {() => Blob} */ let BLOB @@ -54,6 +55,14 @@ async function testContent (input) { await verifyNormalisation(result) } +/** + * @param {*} input + * @param {RegExp} message + */ +async function testFailure (input, message) { + await expect(all(normaliseInput(input))).to.eventually.be.rejectedWith(message) +} + /** * @template T * @param {T} thing @@ -90,14 +99,14 @@ describe('normalise-input-single', function () { /** * @param {() => any} content * @param {string} name - * @param {boolean} isBytes + * @param {{ acceptStream: boolean }} options */ - function testInputType (content, name, isBytes) { + function testInputType (content, name, { acceptStream }) { it(name, async function () { await testContent(content()) }) - if (isBytes) { + if (acceptStream) { if (ReadableStream) { it(`ReadableStream<${name}>`, async function () { await testContent(browserReadableStreamOf(content())) @@ -111,13 +120,27 @@ describe('normalise-input-single', function () { it(`AsyncIterable<${name}>`, async function () { await testContent(asyncIterableOf(content())) }) + } else { + if (ReadableStream) { + it(`Failure ReadableStream<${name}>`, async function () { + await testFailure(browserReadableStreamOf(content()), /Unexpected input/) + }) + } + + it(`Failure Iterable<${name}>`, async function () { + await testFailure(iterableOf(content()), /Unexpected input/) + }) + + it(`Failure AsyncIterable<${name}>`, async function () { + await testFailure(asyncIterableOf(content()), /Unexpected input/) + }) } it(`{ path: '', content: ${name} }`, async function () { await testContent({ path: '', content: content() }) }) - if (isBytes) { + if (acceptStream) { if (ReadableStream) { it(`{ path: '', content: ReadableStream<${name}> }`, async function () { await testContent({ path: '', content: browserReadableStreamOf(content()) }) @@ -134,57 +157,69 @@ describe('normalise-input-single', function () { } if (ReadableStream) { - it(`ReadableStream<${name}>`, async function () { - await testContent(browserReadableStreamOf(content())) - }) + if (acceptStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } else { + it(`Failure ReadableStream<${name}>`, async function () { + await testFailure(browserReadableStreamOf(content()), /multiple items passed/) + }) + } } - it(`Iterable<{ path: '', content: ${name} }`, async function () { - await testContent(iterableOf({ path: '', content: content() })) + it(`Failure Iterable<{ path: '', content: ${name} }>`, async function () { + await testFailure(iterableOf({ path: '', content: content() }), /multiple items passed/) }) - it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { - await testContent(asyncIterableOf({ path: '', content: content() })) + it(`Failure AsyncIterable<{ path: '', content: ${name} }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: content() }), /multiple items passed/) }) - if (isBytes) { + if (acceptStream) { if (ReadableStream) { - it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { - await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + it(`Failure Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: browserReadableStreamOf(content()) }), /multiple items passed/) }) } - it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { - await testContent(iterableOf({ path: '', content: iterableOf(content()) })) + it(`Failure Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: iterableOf(content()) }), /multiple items passed/) }) - it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { - await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) + it(`Failure Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testFailure(iterableOf({ path: '', content: asyncIterableOf(content()) }), /multiple items passed/) }) if (ReadableStream) { - it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { - await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + it(`Failure AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) }), /multiple items passed/) }) } - it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { - await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) + it(`Failure AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: iterableOf(content()) }), /multiple items passed/) }) - it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { - await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content()) })) + it(`Failure AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testFailure(asyncIterableOf({ path: '', content: asyncIterableOf(content()) }), /multiple items passed/) }) } } describe('String', () => { - testInputType(STRING, 'String', true) - testInputType(NEWSTRING, 'new String()', true) + testInputType(STRING, 'String', { + acceptStream: true + }) + testInputType(NEWSTRING, 'new String()', { + acceptStream: true + }) }) describe('Buffer', () => { - testInputType(BUFFER, 'Buffer', true) + testInputType(BUFFER, 'Buffer', { + acceptStream: true + }) }) describe('Blob', () => { @@ -192,23 +227,27 @@ describe('normalise-input-single', function () { return } - testInputType(BLOB, 'Blob', false) + testInputType(BLOB, 'Blob', { + acceptStream: false + }) }) describe('@web-std/file', () => { - it('normalizes File input', async () => { - const FILE = new File([BUFFER()], 'test-file.txt') - - await testContent(FILE) + testInputType(FILE, 'File', { + acceptStream: false }) }) describe('Iterable', () => { - testInputType(ARRAY, 'Iterable', false) + testInputType(ARRAY, 'Iterable', { + acceptStream: false + }) }) describe('TypedArray', () => { - testInputType(TYPEDARRAY, 'TypedArray', true) + testInputType(TYPEDARRAY, 'TypedArray', { + acceptStream: true + }) }) if (isNode) { @@ -226,14 +265,8 @@ describe('normalise-input-single', function () { return fs.createReadStream(path) } - testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) - - it('Iterable', async function () { - await testContent(iterableOf(NODEFSREADSTREAM())) - }) - - it('AsyncIterable', async function () { - await testContent(asyncIterableOf(NODEFSREADSTREAM())) + testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', { + acceptStream: false }) }) } diff --git a/packages/ipfs-core-utils/test/tests.spec.js b/packages/ipfs-core-utils/test/tests.spec.js index 39abcf45fe..cc8a3db62d 100644 --- a/packages/ipfs-core-utils/test/tests.spec.js +++ b/packages/ipfs-core-utils/test/tests.spec.js @@ -1,5 +1,6 @@ import './files/format-mode.spec.js' import './files/format-mtime.spec.js' -import './files/normalise-input.spec.js' +import './files/normalise-input-multiple.spec.js' +import './files/normalise-input-single.spec.js' import './pins/normalise-input.spec.js' From a86b1bea52093fc88328a2f82e5e6b23fc8cccab Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 28 Sep 2021 19:31:46 +0100 Subject: [PATCH 3/5] chore: update interop, use addall for dirs --- packages/interface-ipfs-core/src/add.js | 7 ------- packages/ipfs-core/src/version.js | 4 ++-- packages/ipfs-http-server/src/version.js | 2 +- packages/ipfs/package.json | 2 +- packages/ipfs/src/package.js | 2 +- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index 104e85075e..d9176bd5ff 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -251,13 +251,6 @@ export function testAdd (factory, options) { await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) }) - it('should fail when passed multiple strings', async () => { - const nonValid = ['hello', 'world'] - - // @ts-expect-error nonValid is non valid - await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) - }) - it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } diff --git a/packages/ipfs-core/src/version.js b/packages/ipfs-core/src/version.js index b28eab778b..a5f3719b0a 100644 --- a/packages/ipfs-core/src/version.js +++ b/packages/ipfs-core/src/version.js @@ -1,4 +1,4 @@ -export const ipfsCore = '0.11.0' +export const ipfsCore = '0.11.1' export const commit = '' -export const interfaceIpfsCore = '^0.151.0' +export const interfaceIpfsCore = '^0.151.1' diff --git a/packages/ipfs-http-server/src/version.js b/packages/ipfs-http-server/src/version.js index 83b92fdd3f..f1d7e73424 100644 --- a/packages/ipfs-http-server/src/version.js +++ b/packages/ipfs-http-server/src/version.js @@ -1,2 +1,2 @@ -export const ipfsHttpClient = '^53.0.0' +export const ipfsHttpClient = '^53.0.1' diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 4a36dd6496..e7593aecd1 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -89,7 +89,7 @@ "ipfs-client": "^0.7.1", "ipfs-core-types": "^0.8.1", "ipfs-http-client": "^53.0.1", - "ipfs-interop": "^7.0.1", + "ipfs-interop": "^7.0.2", "ipfs-utils": "^9.0.2", "ipfsd-ctl": "^10.0.4", "iso-url": "^1.0.0", diff --git a/packages/ipfs/src/package.js b/packages/ipfs/src/package.js index 6f97ffa204..988a34d776 100644 --- a/packages/ipfs/src/package.js +++ b/packages/ipfs/src/package.js @@ -1,4 +1,4 @@ export const name = 'ipfs' -export const version = '0.59.0' +export const version = '0.59.1' export const node = '>=14.0.0' From 82ad31f3ea850539f6b8e15a9cf1ea2a8611d320 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 Sep 2021 08:30:13 +0100 Subject: [PATCH 4/5] chore: fix message port client tests --- .../ipfs-message-port-client/package.json | 2 + packages/ipfs-message-port-client/src/core.js | 57 ++++++++++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index fdc4a0a0fb..9e1d1bd866 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -46,9 +46,11 @@ }, "dependencies": { "browser-readablestream-to-it": "^1.0.1", + "err-code": "^3.0.1", "ipfs-core-types": "^0.8.1", "ipfs-message-port-protocol": "^0.10.1", "ipfs-unixfs": "^6.0.3", + "it-peekable": "^1.0.2", "multiformats": "^9.4.1" }, "devDependencies": { diff --git a/packages/ipfs-message-port-client/src/core.js b/packages/ipfs-message-port-client/src/core.js index 768f692053..9e0ea86e1b 100644 --- a/packages/ipfs-message-port-client/src/core.js +++ b/packages/ipfs-message-port-client/src/core.js @@ -16,6 +16,8 @@ import { parseMode, parseMtime } from 'ipfs-unixfs' +import itPeekable from 'it-peekable' +import errCode from 'err-code' /** * @template T @@ -104,7 +106,7 @@ CoreClient.prototype.add = async function add (input, options = {}) { const result = await this.remote.add({ ...options, - input: encodeAddInput(input, transfer), + input: await encodeAddInput(input, transfer), progress: undefined, progressCallback, transfer, @@ -183,9 +185,9 @@ const identity = (v) => v * * @param {ImportCandidate} input * @param {Transferable[]} transfer - * @returns {EncodedAddInput} + * @returns {Promise} */ -const encodeAddInput = (input, transfer) => { +const encodeAddInput = async (input, transfer) => { // We want to get a Blob as input. If we got it we're set. if (input instanceof Blob) { return input @@ -201,13 +203,17 @@ const encodeAddInput = (input, transfer) => { // be encoded via own specific encoder. const iterable = asIterable(input) if (iterable) { - return encodeIterable(iterable, encodeIterableContent, transfer) + return encodeIterable( + await ensureIsByteStream(iterable), + encodeIterableContent, + transfer + ) } const asyncIterable = asAsyncIterable(input) if (asyncIterable) { return encodeIterable( - asyncIterable, + await ensureIsByteStream(asyncIterable), encodeAsyncIterableContent, transfer ) @@ -216,7 +222,7 @@ const encodeAddInput = (input, transfer) => { const readableStream = asReadableStream(input) if (readableStream) { return encodeIterable( - iterateReadableStream(readableStream), + await ensureIsByteStream(iterateReadableStream(readableStream)), encodeAsyncIterableContent, transfer ) @@ -232,7 +238,7 @@ const encodeAddInput = (input, transfer) => { } /** - * Encodes input passed to the `ipfs.add` via the best possible strategy for the + * Encodes input passed to the `ipfs.addAll` via the best possible strategy for the * given input. * * @param {ImportCandidateStream} input @@ -448,3 +454,40 @@ const asFileObject = (input) => { return null } } + +/** + * @template T + * @param {AsyncIterable | Iterable} input + * @returns {Promise | Iterable>} + */ +const ensureIsByteStream = async (input) => { + // @ts-ignore it's (async)iterable + const peekable = itPeekable(input) + + /** @type {any} value **/ + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + return [] + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value) || typeof value === 'string' || value instanceof String) { + return peekable + } + + throw errCode(new Error('Unexpected input: multiple items passed - if you are using ipfs.add, please use ipfs.addAll instead'), 'ERR_UNEXPECTED_INPUT') +} + +/** + * @param {any} obj + * @returns {obj is ArrayBufferView|ArrayBuffer} + */ +function isBytes (obj) { + return ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} From de7616a1e18b085cfd162afc66bc63eb4ce12c59 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 Sep 2021 09:35:49 +0100 Subject: [PATCH 5/5] chore: validate first stream item outside of body loop --- .../src/multipart-request.node.js | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core-utils/src/multipart-request.node.js b/packages/ipfs-core-utils/src/multipart-request.node.js index d406deb50f..d3ad4b6e6f 100644 --- a/packages/ipfs-core-utils/src/multipart-request.node.js +++ b/packages/ipfs-core-utils/src/multipart-request.node.js @@ -5,6 +5,7 @@ import mergeOpts from 'merge-options' // @ts-expect-error no types import toStream from 'it-to-stream' import debug from 'debug' +import itPeekable from 'it-peekable' const merge = mergeOpts.bind({ ignoreUndefined: true }) const log = debug('ipfs:core-utils:multipart-request') @@ -27,8 +28,8 @@ export async function multipartRequest (source, abortController, headers = {}, b try { let index = 0 - // @ts-ignore wrong input type for normaliseInput - for await (const { content, path, mode, mtime } of normaliseInput(source)) { + // @ts-ignore + for await (const { content, path, mode, mtime } of source) { let fileSuffix = '' const type = content ? 'file' : 'dir' @@ -79,12 +80,26 @@ export async function multipartRequest (source, abortController, headers = {}, b } } + // peek at the first value in order to get the input stream moving + // and to validate its contents. + // We cannot do this in the `for await..of` in streamFiles due to + // https://github.com/node-fetch/node-fetch/issues/753 + const peekable = itPeekable(normaliseInput(source)) + + /** @type {any} value **/ + const { value, done } = await peekable.peek() + + if (!done) { + peekable.push(value) + } + return { parts: null, total: -1, headers: merge(headers, { 'Content-Type': `multipart/form-data; boundary=${boundary}` }), - body: await toStream(streamFiles(source)) + // @ts-expect-error normaliseInput returns unixfs importer import candidates + body: toStream(streamFiles(peekable)) } }