diff --git a/README.md b/README.md index 2ac5ccda..213d3a80 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Utility modules to make dealing with async iterators easier, some trivial, some * [it-map](./packages/it-map) Map the output of an iterable * [it-merge](./packages/it-merge) Treat multiple iterables as one * [it-multipart](./packages/it-multipart) Parse multipart message bodies as an iterable +* [it-ndjson](./packages/it-ndjson) Parse multipart message bodies as an iterable * [it-parallel](./packages/it-parallel) Take an iterable of functions that return promises and run them in parallel up to a concurrency limit * [it-parallel-batch](./packages/it-parallel-batch) Take an iterable of functions that return promises and run them in parallel in batches * [it-peekable](./packages/it-peekable) Peek/push an iterable diff --git a/packages/it-ndjson/README.md b/packages/it-ndjson/README.md index 9fbcff61..45b2b995 100644 --- a/packages/it-ndjson/README.md +++ b/packages/it-ndjson/README.md @@ -22,4 +22,8 @@ const values = [0, 1, 2, 3, 4] const arr = await all(ndjson.stringify(values)) console.info(arr) // '0\n', '1\n', '2\n', '3\n', '4\n' + +const res = await all(ndjson.parse(arr)) + +console.info(res) // [0, 1, 2, 3, 4] ``` diff --git a/packages/it-ndjson/package.json b/packages/it-ndjson/package.json index 56f793f1..3d3d6a6f 100644 --- a/packages/it-ndjson/package.json +++ b/packages/it-ndjson/package.json @@ -21,9 +21,10 @@ "devDependencies": { "ava": "^3.12.1", "buffer": "^6.0.3", + "it-all": "^1.0.6", "nyc": "^15.1.0", "standard": "^16.0.3", "typescript": "^4.0.2" }, - "types": "dist/src/index.d.ts" + "types": "dist/index.d.ts" } diff --git a/packages/it-ndjson/parse.js b/packages/it-ndjson/parse.js index dd2aa215..2ea9b21e 100644 --- a/packages/it-ndjson/parse.js +++ b/packages/it-ndjson/parse.js @@ -1,5 +1,5 @@ /** - * @param {AsyncIterable | Iterable} source + * @param {AsyncIterable | Iterable} source */ async function * parse (source) { const matcher = /\r?\n/ diff --git a/packages/it-ndjson/test.js b/packages/it-ndjson/test.js index 67ca26d1..f58a7e4d 100644 --- a/packages/it-ndjson/test.js +++ b/packages/it-ndjson/test.js @@ -1,15 +1,22 @@ const test = require('ava') const ndjson = require('.') const { Buffer } = require('buffer') - -function toAsyncIterator (array) { - return (async function * () { - for (let i = 0; i < array.length; i++) { - yield new Promise(resolve => setTimeout(() => resolve(array[i]))) - } - })() +const all = require('it-all') + +/** + * @template T + * @param {T[]} array + * @returns {AsyncIterable} + */ +async function * toAsyncIterator (array) { + for (let i = 0; i < array.length; i++) { + yield new Promise(resolve => setTimeout(() => resolve(array[i]))) + } } +/** + * @param {string} str + */ function toUint8Array (str) { const arr = new Uint8Array(str.length) for (let i = 0; i < str.length; i++) { @@ -20,77 +27,48 @@ function toUint8Array (str) { test('should split 1 item from 1 chunk', async t => { const source = toAsyncIterator(['{ "id": 1 }\n']) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }]) }) test('should split 1 item from 2 chunks', async t => { const source = toAsyncIterator(['{ "id', '": 1 }\n']) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }]) }) test('should split 2 items from 2 chunks', async t => { const source = toAsyncIterator(['{ "id": 1 }\n', '{ "id": 2 }']) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }, { id: 2 }]) }) test('should split 2 items from 1 chunk', async t => { const source = toAsyncIterator(['{ "id": 1 }\n{ "id": 2 }']) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }, { id: 2 }]) }) test('should split 3 items from 2 chunks', async t => { const source = toAsyncIterator(['{ "id": 1 }\n{ "i', 'd": 2 }', '\n{"id":3}']) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }]) }) test('should split from Buffers', async t => { const source = toAsyncIterator([Buffer.from('{ "id": 1 }\n{ "i'), Buffer.from('d": 2 }'), Buffer.from('\n{"id":3}')]) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } - + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }]) }) test('should split from Uint8Arrays', async t => { const source = toAsyncIterator([toUint8Array('{ "id": 1 }\n{ "i'), toUint8Array('d": 2 }'), toUint8Array('\n{"id":3}')]) - const results = [] - - for await (const value of ndjson.parse(source)) { - results.push(value) - } + const results = await all(ndjson.parse(source)) t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }]) }) @@ -98,11 +76,15 @@ test('should split from Uint8Arrays', async t => { test('should round trip', async t => { const input = '{"id":1}\n{"id":2}\n{"id":3}\n' const source = toAsyncIterator([input]) - const results = [] - - for await (const value of ndjson.stringify(ndjson.parse(source))) { - results.push(value) - } + const results = await all(ndjson.stringify(ndjson.parse(source))) t.is(results.join(''), input) }) + +test('should stringify trip', async t => { + const input = [{ id: 1 }, { id: 2 }, { id: 3 }] + const source = toAsyncIterator(input) + const results = await all(ndjson.stringify(source)) + + t.is(results.join(''), '{"id":1}\n{"id":2}\n{"id":3}\n') +})