From df751d9b8bbab8a77ad996bb7168b70a70310129 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 02:24:38 +0300 Subject: [PATCH 01/11] test: ensure stream preprocessing order Sometimes it is necessary to preprocess some initial bit of a stream data before giving the entire stream to the main processing function. Sometimes this bit should be extracted from the stream before the main processing; sometimes it should be returned to the stream. This test checks an order of stream modes, methods and events for a possible preprocessing algorithm. Stream BOM stripping is selected as a use case. See https://github.com/nodejs/help/issues/221 as the prehistory. --- .../file-to-read-with-bom.txt | 3 + .../file-to-read-without-bom.txt | 3 + .../stream-preprocess/stream-preprocess.js | 69 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 test/fixtures/stream-preprocess/file-to-read-with-bom.txt create mode 100644 test/fixtures/stream-preprocess/file-to-read-without-bom.txt create mode 100644 test/fixtures/stream-preprocess/stream-preprocess.js diff --git a/test/fixtures/stream-preprocess/file-to-read-with-bom.txt b/test/fixtures/stream-preprocess/file-to-read-with-bom.txt new file mode 100644 index 00000000000000..d46c8708d97946 --- /dev/null +++ b/test/fixtures/stream-preprocess/file-to-read-with-bom.txt @@ -0,0 +1,3 @@ +abc +def +ghi diff --git a/test/fixtures/stream-preprocess/file-to-read-without-bom.txt b/test/fixtures/stream-preprocess/file-to-read-without-bom.txt new file mode 100644 index 00000000000000..8edb37e36dfbc0 --- /dev/null +++ b/test/fixtures/stream-preprocess/file-to-read-without-bom.txt @@ -0,0 +1,3 @@ +abc +def +ghi diff --git a/test/fixtures/stream-preprocess/stream-preprocess.js b/test/fixtures/stream-preprocess/stream-preprocess.js new file mode 100644 index 00000000000000..4387afdeef8461 --- /dev/null +++ b/test/fixtures/stream-preprocess/stream-preprocess.js @@ -0,0 +1,69 @@ +/******************************************************************************/ +'use strict'; +/******************************************************************************/ +const common = require('../../common'); +const assert = require('assert'); + +const fs = require('fs'); +const path = require('path'); +const rl = require('readline'); +/******************************************************************************/ +const BOM = '\uFEFF'; + +// get the data by a non-stream way to compare with the streamed data +const modelData = fs.readFileSync( + path.join(__dirname, 'file-to-read-without-bom.txt'), 'utf8' +); +const modelDataFirstCharacter = modelData[0]; + +// detect the number of forthcoming 'line' events for mustCall() 'expected' arg +const linesNumber = modelData.match(/\n/g).length; +/******************************************************************************/ +// ensure both without-bom and with-bom test files are textwise equal +assert.strictEqual( + fs.readFileSync(path.join(__dirname, 'file-to-read-with-bom.txt'), 'utf8'), + `${BOM}${modelData}` +); +/******************************************************************************/ +// an unjustified BOM stripping with a non-BOM character unshifted to a stream +const inputWithoutBOM = fs.createReadStream( + path.join(__dirname, 'file-to-read-without-bom.txt'), 'utf8' +); + +inputWithoutBOM.once('readable', common.mustCall(() => { + const maybeBOM = inputWithoutBOM.read(1); + assert.strictEqual(maybeBOM, modelDataFirstCharacter); + + if (maybeBOM !== BOM) inputWithoutBOM.unshift(maybeBOM); + + let streamedData = ''; + rl.createInterface({ + input: inputWithoutBOM, + }).on('line', common.mustCall((line) => { + streamedData += `${line}\n`; + }, linesNumber)).on('close', common.mustCall(() => { + assert.strictEqual(streamedData, modelData); + })); +})); +/******************************************************************************/ +// a justified BOM stripping +const inputWithBOM = fs.createReadStream( + path.join(__dirname, 'file-to-read-with-bom.txt'), 'utf8' +); + +inputWithBOM.once('readable', common.mustCall(() => { + const maybeBOM = inputWithBOM.read(1); + assert.strictEqual(maybeBOM, BOM); + + if (maybeBOM !== BOM) inputWithBOM.unshift(maybeBOM); + + let streamedData = ''; + rl.createInterface({ + input: inputWithBOM, + }).on('line', common.mustCall((line) => { + streamedData += `${line}\n`; + }, linesNumber)).on('close', common.mustCall(() => { + assert.strictEqual(streamedData, modelData); + })); +})); +/******************************************************************************/ From 50fd2c84fda050462fbfdc23fee66f9bb08e6ed6 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 03:39:13 +0300 Subject: [PATCH 02/11] Delete stream-preprocess.js --- .../stream-preprocess/stream-preprocess.js | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 test/fixtures/stream-preprocess/stream-preprocess.js diff --git a/test/fixtures/stream-preprocess/stream-preprocess.js b/test/fixtures/stream-preprocess/stream-preprocess.js deleted file mode 100644 index 4387afdeef8461..00000000000000 --- a/test/fixtures/stream-preprocess/stream-preprocess.js +++ /dev/null @@ -1,69 +0,0 @@ -/******************************************************************************/ -'use strict'; -/******************************************************************************/ -const common = require('../../common'); -const assert = require('assert'); - -const fs = require('fs'); -const path = require('path'); -const rl = require('readline'); -/******************************************************************************/ -const BOM = '\uFEFF'; - -// get the data by a non-stream way to compare with the streamed data -const modelData = fs.readFileSync( - path.join(__dirname, 'file-to-read-without-bom.txt'), 'utf8' -); -const modelDataFirstCharacter = modelData[0]; - -// detect the number of forthcoming 'line' events for mustCall() 'expected' arg -const linesNumber = modelData.match(/\n/g).length; -/******************************************************************************/ -// ensure both without-bom and with-bom test files are textwise equal -assert.strictEqual( - fs.readFileSync(path.join(__dirname, 'file-to-read-with-bom.txt'), 'utf8'), - `${BOM}${modelData}` -); -/******************************************************************************/ -// an unjustified BOM stripping with a non-BOM character unshifted to a stream -const inputWithoutBOM = fs.createReadStream( - path.join(__dirname, 'file-to-read-without-bom.txt'), 'utf8' -); - -inputWithoutBOM.once('readable', common.mustCall(() => { - const maybeBOM = inputWithoutBOM.read(1); - assert.strictEqual(maybeBOM, modelDataFirstCharacter); - - if (maybeBOM !== BOM) inputWithoutBOM.unshift(maybeBOM); - - let streamedData = ''; - rl.createInterface({ - input: inputWithoutBOM, - }).on('line', common.mustCall((line) => { - streamedData += `${line}\n`; - }, linesNumber)).on('close', common.mustCall(() => { - assert.strictEqual(streamedData, modelData); - })); -})); -/******************************************************************************/ -// a justified BOM stripping -const inputWithBOM = fs.createReadStream( - path.join(__dirname, 'file-to-read-with-bom.txt'), 'utf8' -); - -inputWithBOM.once('readable', common.mustCall(() => { - const maybeBOM = inputWithBOM.read(1); - assert.strictEqual(maybeBOM, BOM); - - if (maybeBOM !== BOM) inputWithBOM.unshift(maybeBOM); - - let streamedData = ''; - rl.createInterface({ - input: inputWithBOM, - }).on('line', common.mustCall((line) => { - streamedData += `${line}\n`; - }, linesNumber)).on('close', common.mustCall(() => { - assert.strictEqual(streamedData, modelData); - })); -})); -/******************************************************************************/ From 06f89b847b45e2de8caffb5fcc899e42ab16dba1 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 03:39:22 +0300 Subject: [PATCH 03/11] Delete file-to-read-without-bom.txt --- test/fixtures/stream-preprocess/file-to-read-without-bom.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 test/fixtures/stream-preprocess/file-to-read-without-bom.txt diff --git a/test/fixtures/stream-preprocess/file-to-read-without-bom.txt b/test/fixtures/stream-preprocess/file-to-read-without-bom.txt deleted file mode 100644 index 8edb37e36dfbc0..00000000000000 --- a/test/fixtures/stream-preprocess/file-to-read-without-bom.txt +++ /dev/null @@ -1,3 +0,0 @@ -abc -def -ghi From d74e6dfe56c427fa7092188a5ba30fbaf3e56612 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 03:39:30 +0300 Subject: [PATCH 04/11] Delete file-to-read-with-bom.txt --- test/fixtures/stream-preprocess/file-to-read-with-bom.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 test/fixtures/stream-preprocess/file-to-read-with-bom.txt diff --git a/test/fixtures/stream-preprocess/file-to-read-with-bom.txt b/test/fixtures/stream-preprocess/file-to-read-with-bom.txt deleted file mode 100644 index d46c8708d97946..00000000000000 --- a/test/fixtures/stream-preprocess/file-to-read-with-bom.txt +++ /dev/null @@ -1,3 +0,0 @@ -abc -def -ghi From 62379f3ebd157a2f9da12678fc64f56313ca7abb Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 03:40:36 +0300 Subject: [PATCH 05/11] Add .txt files to fixtures --- test/fixtures/file-to-read-with-bom.txt | 3 +++ test/fixtures/file-to-read-without-bom.txt | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 test/fixtures/file-to-read-with-bom.txt create mode 100644 test/fixtures/file-to-read-without-bom.txt diff --git a/test/fixtures/file-to-read-with-bom.txt b/test/fixtures/file-to-read-with-bom.txt new file mode 100644 index 00000000000000..d46c8708d97946 --- /dev/null +++ b/test/fixtures/file-to-read-with-bom.txt @@ -0,0 +1,3 @@ +abc +def +ghi diff --git a/test/fixtures/file-to-read-without-bom.txt b/test/fixtures/file-to-read-without-bom.txt new file mode 100644 index 00000000000000..8edb37e36dfbc0 --- /dev/null +++ b/test/fixtures/file-to-read-without-bom.txt @@ -0,0 +1,3 @@ +abc +def +ghi From 099a1110face1b14885249ea966c756e5800b4d5 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 03:41:32 +0300 Subject: [PATCH 06/11] Add test-stream-preprocess.js to parallel --- test/parallel/test-stream-preprocess.js | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/parallel/test-stream-preprocess.js diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js new file mode 100644 index 00000000000000..e87561fdc348b0 --- /dev/null +++ b/test/parallel/test-stream-preprocess.js @@ -0,0 +1,71 @@ +/******************************************************************************/ +'use strict'; +/******************************************************************************/ +const common = require('../common'); +const assert = require('assert'); + +const fs = require('fs'); +const path = require('path'); +const rl = require('readline'); +/******************************************************************************/ +const BOM = '\uFEFF'; + +// get the data by a non-stream way to compare with the streamed data +const modelData = fs.readFileSync( + path.join(__dirname, '..', 'fixtures', 'file-to-read-without-bom.txt'), 'utf8' +); +const modelDataFirstCharacter = modelData[0]; + +// detect the number of forthcoming 'line' events for mustCall() 'expected' arg +const linesNumber = modelData.match(/\n/g).length; +/******************************************************************************/ +// ensure both without-bom and with-bom test files are textwise equal +assert.strictEqual( + fs.readFileSync( + path.join(__dirname, '..', 'fixtures', 'file-to-read-with-bom.txt'), 'utf8' + ), + `${BOM}${modelData}` +); +/******************************************************************************/ +// an unjustified BOM stripping with a non-BOM character unshifted to a stream +const inputWithoutBOM = fs.createReadStream( + path.join(__dirname, '..', 'fixtures', 'file-to-read-without-bom.txt'), 'utf8' +); + +inputWithoutBOM.once('readable', common.mustCall(() => { + const maybeBOM = inputWithoutBOM.read(1); + assert.strictEqual(maybeBOM, modelDataFirstCharacter); + + if (maybeBOM !== BOM) inputWithoutBOM.unshift(maybeBOM); + + let streamedData = ''; + rl.createInterface({ + input: inputWithoutBOM, + }).on('line', common.mustCall((line) => { + streamedData += `${line}\n`; + }, linesNumber)).on('close', common.mustCall(() => { + assert.strictEqual(streamedData, modelData); + })); +})); +/******************************************************************************/ +// a justified BOM stripping +const inputWithBOM = fs.createReadStream( + path.join(__dirname, '..', 'fixtures', 'file-to-read-with-bom.txt'), 'utf8' +); + +inputWithBOM.once('readable', common.mustCall(() => { + const maybeBOM = inputWithBOM.read(1); + assert.strictEqual(maybeBOM, BOM); + + if (maybeBOM !== BOM) inputWithBOM.unshift(maybeBOM); + + let streamedData = ''; + rl.createInterface({ + input: inputWithBOM, + }).on('line', common.mustCall((line) => { + streamedData += `${line}\n`; + }, linesNumber)).on('close', common.mustCall(() => { + assert.strictEqual(streamedData, modelData); + })); +})); +/******************************************************************************/ From a05ea8322f9af7849032b9409d7b5f9a05da6c5c Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 04:29:35 +0300 Subject: [PATCH 07/11] Fix test-stream-preprocess.js style issues --- test/parallel/test-stream-preprocess.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js index e87561fdc348b0..9386abd34a0559 100644 --- a/test/parallel/test-stream-preprocess.js +++ b/test/parallel/test-stream-preprocess.js @@ -1,13 +1,12 @@ -/******************************************************************************/ 'use strict'; -/******************************************************************************/ + const common = require('../common'); const assert = require('assert'); const fs = require('fs'); const path = require('path'); const rl = require('readline'); -/******************************************************************************/ + const BOM = '\uFEFF'; // get the data by a non-stream way to compare with the streamed data @@ -18,7 +17,7 @@ const modelDataFirstCharacter = modelData[0]; // detect the number of forthcoming 'line' events for mustCall() 'expected' arg const linesNumber = modelData.match(/\n/g).length; -/******************************************************************************/ + // ensure both without-bom and with-bom test files are textwise equal assert.strictEqual( fs.readFileSync( @@ -26,7 +25,7 @@ assert.strictEqual( ), `${BOM}${modelData}` ); -/******************************************************************************/ + // an unjustified BOM stripping with a non-BOM character unshifted to a stream const inputWithoutBOM = fs.createReadStream( path.join(__dirname, '..', 'fixtures', 'file-to-read-without-bom.txt'), 'utf8' @@ -47,7 +46,7 @@ inputWithoutBOM.once('readable', common.mustCall(() => { assert.strictEqual(streamedData, modelData); })); })); -/******************************************************************************/ + // a justified BOM stripping const inputWithBOM = fs.createReadStream( path.join(__dirname, '..', 'fixtures', 'file-to-read-with-bom.txt'), 'utf8' @@ -68,4 +67,3 @@ inputWithBOM.once('readable', common.mustCall(() => { assert.strictEqual(streamedData, modelData); })); })); -/******************************************************************************/ From 473da056ea96bf933ed5eeb6d30d55386495d733 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 04:32:53 +0300 Subject: [PATCH 08/11] Fix another test-stream-preprocess.js style issue --- test/parallel/test-stream-preprocess.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js index 9386abd34a0559..a2c267f71fb0ae 100644 --- a/test/parallel/test-stream-preprocess.js +++ b/test/parallel/test-stream-preprocess.js @@ -1,5 +1,4 @@ 'use strict'; - const common = require('../common'); const assert = require('assert'); From 376ea476de7bcbd5d2fd4d27152edc0c47f9acb7 Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 18:55:42 +0300 Subject: [PATCH 09/11] Use common.fixturesDir, add some more assertions --- test/parallel/test-stream-preprocess.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js index a2c267f71fb0ae..a1ba3d11dec9a4 100644 --- a/test/parallel/test-stream-preprocess.js +++ b/test/parallel/test-stream-preprocess.js @@ -10,7 +10,7 @@ const BOM = '\uFEFF'; // get the data by a non-stream way to compare with the streamed data const modelData = fs.readFileSync( - path.join(__dirname, '..', 'fixtures', 'file-to-read-without-bom.txt'), 'utf8' + path.join(common.fixturesDir, 'file-to-read-without-bom.txt'), 'utf8' ); const modelDataFirstCharacter = modelData[0]; @@ -20,21 +20,22 @@ const linesNumber = modelData.match(/\n/g).length; // ensure both without-bom and with-bom test files are textwise equal assert.strictEqual( fs.readFileSync( - path.join(__dirname, '..', 'fixtures', 'file-to-read-with-bom.txt'), 'utf8' + path.join(common.fixturesDir, 'file-to-read-with-bom.txt'), 'utf8' ), `${BOM}${modelData}` ); // an unjustified BOM stripping with a non-BOM character unshifted to a stream const inputWithoutBOM = fs.createReadStream( - path.join(__dirname, '..', 'fixtures', 'file-to-read-without-bom.txt'), 'utf8' + path.join(common.fixturesDir, 'file-to-read-without-bom.txt'), 'utf8' ); inputWithoutBOM.once('readable', common.mustCall(() => { const maybeBOM = inputWithoutBOM.read(1); assert.strictEqual(maybeBOM, modelDataFirstCharacter); - if (maybeBOM !== BOM) inputWithoutBOM.unshift(maybeBOM); + assert.notStrictEqual(maybeBOM, BOM); + inputWithoutBOM.unshift(maybeBOM); let streamedData = ''; rl.createInterface({ @@ -48,14 +49,14 @@ inputWithoutBOM.once('readable', common.mustCall(() => { // a justified BOM stripping const inputWithBOM = fs.createReadStream( - path.join(__dirname, '..', 'fixtures', 'file-to-read-with-bom.txt'), 'utf8' + path.join(common.fixturesDir, 'file-to-read-with-bom.txt'), 'utf8' ); inputWithBOM.once('readable', common.mustCall(() => { const maybeBOM = inputWithBOM.read(1); assert.strictEqual(maybeBOM, BOM); - if (maybeBOM !== BOM) inputWithBOM.unshift(maybeBOM); + assert.strictEqual(maybeBOM, BOM); let streamedData = ''; rl.createInterface({ From f41dca2ef6c8d2a09c671705e97489d557feecee Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Fri, 15 Jul 2016 19:01:06 +0300 Subject: [PATCH 10/11] Remove redundant assertions --- test/parallel/test-stream-preprocess.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js index a1ba3d11dec9a4..bab889b358b6e2 100644 --- a/test/parallel/test-stream-preprocess.js +++ b/test/parallel/test-stream-preprocess.js @@ -33,8 +33,8 @@ const inputWithoutBOM = fs.createReadStream( inputWithoutBOM.once('readable', common.mustCall(() => { const maybeBOM = inputWithoutBOM.read(1); assert.strictEqual(maybeBOM, modelDataFirstCharacter); - assert.notStrictEqual(maybeBOM, BOM); + inputWithoutBOM.unshift(maybeBOM); let streamedData = ''; @@ -56,8 +56,6 @@ inputWithBOM.once('readable', common.mustCall(() => { const maybeBOM = inputWithBOM.read(1); assert.strictEqual(maybeBOM, BOM); - assert.strictEqual(maybeBOM, BOM); - let streamedData = ''; rl.createInterface({ input: inputWithBOM, From 6d7e25fbd43056283ca54449dd93b71bed3fffad Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Wed, 20 Jul 2016 02:25:29 +0300 Subject: [PATCH 11/11] Fix comments style and variable name --- test/parallel/test-stream-preprocess.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/parallel/test-stream-preprocess.js b/test/parallel/test-stream-preprocess.js index bab889b358b6e2..9cb4c1d3cca8ae 100644 --- a/test/parallel/test-stream-preprocess.js +++ b/test/parallel/test-stream-preprocess.js @@ -8,16 +8,16 @@ const rl = require('readline'); const BOM = '\uFEFF'; -// get the data by a non-stream way to compare with the streamed data +// Get the data using a non-stream way to compare with the streamed data. const modelData = fs.readFileSync( path.join(common.fixturesDir, 'file-to-read-without-bom.txt'), 'utf8' ); const modelDataFirstCharacter = modelData[0]; -// detect the number of forthcoming 'line' events for mustCall() 'expected' arg -const linesNumber = modelData.match(/\n/g).length; +// Detect the number of forthcoming 'line' events for mustCall() 'expected' arg. +const lineCount = modelData.match(/\n/g).length; -// ensure both without-bom and with-bom test files are textwise equal +// Ensure both without-bom and with-bom test files are textwise equal. assert.strictEqual( fs.readFileSync( path.join(common.fixturesDir, 'file-to-read-with-bom.txt'), 'utf8' @@ -25,7 +25,7 @@ assert.strictEqual( `${BOM}${modelData}` ); -// an unjustified BOM stripping with a non-BOM character unshifted to a stream +// An unjustified BOM stripping with a non-BOM character unshifted to a stream. const inputWithoutBOM = fs.createReadStream( path.join(common.fixturesDir, 'file-to-read-without-bom.txt'), 'utf8' ); @@ -42,12 +42,12 @@ inputWithoutBOM.once('readable', common.mustCall(() => { input: inputWithoutBOM, }).on('line', common.mustCall((line) => { streamedData += `${line}\n`; - }, linesNumber)).on('close', common.mustCall(() => { + }, lineCount)).on('close', common.mustCall(() => { assert.strictEqual(streamedData, modelData); })); })); -// a justified BOM stripping +// A justified BOM stripping. const inputWithBOM = fs.createReadStream( path.join(common.fixturesDir, 'file-to-read-with-bom.txt'), 'utf8' ); @@ -61,7 +61,7 @@ inputWithBOM.once('readable', common.mustCall(() => { input: inputWithBOM, }).on('line', common.mustCall((line) => { streamedData += `${line}\n`; - }, linesNumber)).on('close', common.mustCall(() => { + }, lineCount)).on('close', common.mustCall(() => { assert.strictEqual(streamedData, modelData); })); }));