From 807ee4007861cb5e93cf4a9bcc850a5dd88321d6 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 28 May 2019 14:01:05 +0200 Subject: [PATCH] stream: convert existing buffer when calling .setEncoding Convert already-stored chunks when `.setEncoding()` is called so that subsequent `data` events will receive decoded strings, as they expect. Fixes: https://github.com/nodejs/node/issues/27932 PR-URL: https://github.com/nodejs/node/pull/27936 Reviewed-By: Matteo Collina Reviewed-By: Luigi Pinca Reviewed-By: Ruben Bridgewater Reviewed-By: Rich Trott --- lib/_stream_readable.js | 17 +++++- ...m-readable-setEncoding-existing-buffers.js | 60 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 test/parallel/test-stream-readable-setEncoding-existing-buffers.js diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index a000c9390e31ac..03f7924e5c5a3c 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -321,9 +321,22 @@ Readable.prototype.isPaused = function() { Readable.prototype.setEncoding = function(enc) { if (!StringDecoder) StringDecoder = require('string_decoder').StringDecoder; - this._readableState.decoder = new StringDecoder(enc); - // if setEncoding(null), decoder.encoding equals utf8 + const decoder = new StringDecoder(enc); + this._readableState.decoder = decoder; + // If setEncoding(null), decoder.encoding equals utf8 this._readableState.encoding = this._readableState.decoder.encoding; + + // Iterate over current buffer to convert already stored Buffers: + let p = this._readableState.buffer.head; + let content = ''; + while (p !== null) { + content += decoder.write(p.data); + p = p.next; + } + this._readableState.buffer.clear(); + if (content !== '') + this._readableState.buffer.push(content); + this._readableState.length = content.length; return this; }; diff --git a/test/parallel/test-stream-readable-setEncoding-existing-buffers.js b/test/parallel/test-stream-readable-setEncoding-existing-buffers.js new file mode 100644 index 00000000000000..eb75260bacfc45 --- /dev/null +++ b/test/parallel/test-stream-readable-setEncoding-existing-buffers.js @@ -0,0 +1,60 @@ +'use strict'; +require('../common'); +const { Readable } = require('stream'); +const assert = require('assert'); + +{ + // Call .setEncoding() while there are bytes already in the buffer. + const r = new Readable({ read() {} }); + + r.push(Buffer.from('a')); + r.push(Buffer.from('b')); + + r.setEncoding('utf8'); + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['ab']); + }); +} + +{ + // Call .setEncoding() while the buffer contains a complete, + // but chunked character. + const r = new Readable({ read() {} }); + + r.push(Buffer.from([0xf0])); + r.push(Buffer.from([0x9f])); + r.push(Buffer.from([0x8e])); + r.push(Buffer.from([0x89])); + + r.setEncoding('utf8'); + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['🎉']); + }); +} + +{ + // Call .setEncoding() while the buffer contains an incomplete character, + // and finish the character later. + const r = new Readable({ read() {} }); + + r.push(Buffer.from([0xf0])); + r.push(Buffer.from([0x9f])); + + r.setEncoding('utf8'); + + r.push(Buffer.from([0x8e])); + r.push(Buffer.from([0x89])); + + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['🎉']); + }); +}