From a7677bc352551b722d7ebf1a1ce860e02b3868ae Mon Sep 17 00:00:00 2001 From: Livia Medeiros <74449973+LiviaMedeiros@users.noreply.github.com> Date: Tue, 19 Jul 2022 00:35:14 +0900 Subject: [PATCH 01/46] fs: use signed types for stat data This allows to support timestamps before 1970-01-01. On Windows, it's not supported due to Y2038 issue. PR-URL: https://github.com/nodejs/node/pull/43714 Fixes: https://github.com/nodejs/node/issues/43707 Reviewed-By: Ben Noordhuis Reviewed-By: Antoine du Hamel --- lib/internal/fs/utils.js | 17 ++++--- src/aliased_buffer.h | 2 +- src/node_file-inl.h | 19 +++++-- src/node_file.h | 2 +- test/parallel/test-fs-stat-date.mjs | 79 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 test/parallel/test-fs-stat-date.mjs diff --git a/lib/internal/fs/utils.js b/lib/internal/fs/utils.js index 9280fcfef7018e..2e5e21342c8797 100644 --- a/lib/internal/fs/utils.js +++ b/lib/internal/fs/utils.js @@ -12,6 +12,7 @@ const { NumberIsFinite, NumberIsInteger, MathMin, + MathRound, ObjectIs, ObjectPrototypeHasOwnProperty, ObjectSetPrototypeOf, @@ -39,9 +40,9 @@ const { } = require('internal/errors'); const { isArrayBufferView, - isUint8Array, + isBigInt64Array, isDate, - isBigUint64Array + isUint8Array, } = require('internal/util/types'); const { kEmptyObject, @@ -454,14 +455,16 @@ function nsFromTimeSpecBigInt(sec, nsec) { return sec * kNsPerSecBigInt + nsec; } -// The Date constructor performs Math.floor() to the timestamp. -// https://www.ecma-international.org/ecma-262/#sec-timeclip +// The Date constructor performs Math.floor() on the absolute value +// of the timestamp: https://tc39.es/ecma262/#sec-timeclip // Since there may be a precision loss when the timestamp is // converted to a floating point number, we manually round // the timestamp here before passing it to Date(). // Refs: https://github.com/nodejs/node/pull/12607 +// Refs: https://github.com/nodejs/node/pull/43714 function dateFromMs(ms) { - return new Date(Number(ms) + 0.5); + // Coercing to number, ms can be bigint + return new Date(MathRound(Number(ms))); } function BigIntStats(dev, mode, nlink, uid, gid, rdev, blksize, @@ -526,12 +529,12 @@ Stats.prototype._checkModeProperty = function(property) { }; /** - * @param {Float64Array | BigUint64Array} stats + * @param {Float64Array | BigInt64Array} stats * @param {number} offset * @returns {BigIntStats | Stats} */ function getStatsFromBinding(stats, offset = 0) { - if (isBigUint64Array(stats)) { + if (isBigInt64Array(stats)) { return new BigIntStats( stats[0 + offset], stats[1 + offset], stats[2 + offset], stats[3 + offset], stats[4 + offset], stats[5 + offset], diff --git a/src/aliased_buffer.h b/src/aliased_buffer.h index 2cbc70aaa7c37c..6dda51c14615cc 100644 --- a/src/aliased_buffer.h +++ b/src/aliased_buffer.h @@ -307,7 +307,7 @@ typedef AliasedBufferBase AliasedInt32Array; typedef AliasedBufferBase AliasedUint8Array; typedef AliasedBufferBase AliasedUint32Array; typedef AliasedBufferBase AliasedFloat64Array; -typedef AliasedBufferBase AliasedBigUint64Array; +typedef AliasedBufferBase AliasedBigInt64Array; } // namespace node #endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS diff --git a/src/node_file-inl.h b/src/node_file-inl.h index 351f3df809d94a..1170d57754ffad 100644 --- a/src/node_file-inl.h +++ b/src/node_file-inl.h @@ -86,13 +86,22 @@ template void FillStatsArray(AliasedBufferBase* fields, const uv_stat_t* s, const size_t offset) { -#define SET_FIELD_WITH_STAT(stat_offset, stat) \ - fields->SetValue(offset + static_cast(FsStatsOffset::stat_offset), \ +#define SET_FIELD_WITH_STAT(stat_offset, stat) \ + fields->SetValue(offset + static_cast(FsStatsOffset::stat_offset), \ static_cast(stat)) -#define SET_FIELD_WITH_TIME_STAT(stat_offset, stat) \ - /* NOLINTNEXTLINE(runtime/int) */ \ +// On win32, time is stored in uint64_t and starts from 1601-01-01. +// libuv calculates tv_sec and tv_nsec from it and converts to signed long, +// which causes Y2038 overflow. On the other platforms it is safe to treat +// negative values as pre-epoch time. +#ifdef _WIN32 +#define SET_FIELD_WITH_TIME_STAT(stat_offset, stat) \ + /* NOLINTNEXTLINE(runtime/int) */ \ SET_FIELD_WITH_STAT(stat_offset, static_cast(stat)) +#else +#define SET_FIELD_WITH_TIME_STAT(stat_offset, stat) \ + SET_FIELD_WITH_STAT(stat_offset, static_cast(stat)) +#endif // _WIN32 SET_FIELD_WITH_STAT(kDev, s->st_dev); SET_FIELD_WITH_STAT(kMode, s->st_mode); @@ -233,7 +242,7 @@ FSReqBase* GetReqWrap(const v8::FunctionCallbackInfo& args, Environment* env = binding_data->env(); if (value->StrictEquals(env->fs_use_promises_symbol())) { if (use_bigint) { - return FSReqPromise::New(binding_data, use_bigint); + return FSReqPromise::New(binding_data, use_bigint); } else { return FSReqPromise::New(binding_data, use_bigint); } diff --git a/src/node_file.h b/src/node_file.h index cf98c5c933bb84..9d2834fa2673d6 100644 --- a/src/node_file.h +++ b/src/node_file.h @@ -18,7 +18,7 @@ class BindingData : public SnapshotableObject { explicit BindingData(Environment* env, v8::Local wrap); AliasedFloat64Array stats_field_array; - AliasedBigUint64Array stats_field_bigint_array; + AliasedBigInt64Array stats_field_bigint_array; std::vector> file_handle_read_wrap_freelist; diff --git a/test/parallel/test-fs-stat-date.mjs b/test/parallel/test-fs-stat-date.mjs new file mode 100644 index 00000000000000..c3b52f070cab18 --- /dev/null +++ b/test/parallel/test-fs-stat-date.mjs @@ -0,0 +1,79 @@ +import * as common from '../common/index.mjs'; + +// Test timestamps returned by fsPromises.stat and fs.statSync + +import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; +import path from 'node:path'; +import assert from 'node:assert'; +import tmpdir from '../common/tmpdir.js'; + +// On some platforms (for example, ppc64) boundaries are tighter +// than usual. If we catch these errors, skip corresponding test. +const ignoredErrors = new Set(['EINVAL', 'EOVERFLOW']); + +tmpdir.refresh(); +const filepath = path.resolve(tmpdir.path, 'timestamp'); + +await (await fsPromises.open(filepath, 'w')).close(); + +// Date might round down timestamp +function closeEnough(actual, expected, margin) { + // On ppc64, value is rounded to seconds + if (process.arch === 'ppc64') { + margin += 1000; + } + assert.ok(Math.abs(Number(actual - expected)) < margin, + `expected ${expected} ± ${margin}, got ${actual}`); +} + +async function runTest(atime, mtime, margin = 0) { + margin += Number.EPSILON; + try { + await fsPromises.utimes(filepath, new Date(atime), new Date(mtime)); + } catch (e) { + if (ignoredErrors.has(e.code)) return; + throw e; + } + + const stats = await fsPromises.stat(filepath); + closeEnough(stats.atimeMs, atime, margin); + closeEnough(stats.mtimeMs, mtime, margin); + closeEnough(stats.atime.getTime(), new Date(atime).getTime(), margin); + closeEnough(stats.mtime.getTime(), new Date(mtime).getTime(), margin); + + const statsBigint = await fsPromises.stat(filepath, { bigint: true }); + closeEnough(statsBigint.atimeMs, BigInt(atime), margin); + closeEnough(statsBigint.mtimeMs, BigInt(mtime), margin); + closeEnough(statsBigint.atime.getTime(), new Date(atime).getTime(), margin); + closeEnough(statsBigint.mtime.getTime(), new Date(mtime).getTime(), margin); + + const statsSync = fs.statSync(filepath); + closeEnough(statsSync.atimeMs, atime, margin); + closeEnough(statsSync.mtimeMs, mtime, margin); + closeEnough(statsSync.atime.getTime(), new Date(atime).getTime(), margin); + closeEnough(statsSync.mtime.getTime(), new Date(mtime).getTime(), margin); + + const statsSyncBigint = fs.statSync(filepath, { bigint: true }); + closeEnough(statsSyncBigint.atimeMs, BigInt(atime), margin); + closeEnough(statsSyncBigint.mtimeMs, BigInt(mtime), margin); + closeEnough(statsSyncBigint.atime.getTime(), new Date(atime).getTime(), margin); + closeEnough(statsSyncBigint.mtime.getTime(), new Date(mtime).getTime(), margin); +} + +// Too high/low numbers produce too different results on different platforms +{ + // TODO(LiviaMedeiros): investigate outdated stat time on FreeBSD. + // On Windows, filetime is stored and handled differently. Supporting dates + // after Y2038 is preferred over supporting dates before 1970-01-01. + if (!common.isFreeBSD && !common.isWindows) { + await runTest(-40691, -355, 1); // Potential precision loss on 32bit + await runTest(-355, -40691, 1); // Potential precision loss on 32bit + await runTest(-1, -1); + } + await runTest(0, 0); + await runTest(1, 1); + await runTest(355, 40691, 1); // Precision loss on 32bit + await runTest(40691, 355, 1); // Precision loss on 32bit + await runTest(1713037251360, 1713037251360, 1); // Precision loss +} From b8e5873df8f073e78dd8c726290767e874550a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Nie=C3=9Fen?= Date: Mon, 18 Jul 2022 18:29:40 +0200 Subject: [PATCH 02/46] src: use named struct instead of typedef PR-URL: https://github.com/nodejs/node/pull/43881 Reviewed-By: Ben Noordhuis Reviewed-By: Darshan Sen Reviewed-By: Antoine du Hamel Reviewed-By: Filip Skokan Reviewed-By: Ruben Bridgewater --- src/crypto/crypto_groups.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto/crypto_groups.h b/src/crypto/crypto_groups.h index c49b4356d8a919..c43569db5e6c8e 100644 --- a/src/crypto/crypto_groups.h +++ b/src/crypto/crypto_groups.h @@ -390,12 +390,12 @@ static const unsigned char group_modp18[] = { 0x80, 0xdd, 0x98, 0xed, 0xd3, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -typedef struct { +struct modp_group { const char* name; const char* prime; unsigned int prime_size; unsigned int gen; -} modp_group; +}; static const modp_group modp_groups[] = { #define V(var) reinterpret_cast(var) From cc97a8d443b579750d4001035932fa2a0dc8e477 Mon Sep 17 00:00:00 2001 From: "Node.js GitHub Bot" Date: Mon, 18 Jul 2022 19:09:58 +0200 Subject: [PATCH 03/46] meta: update AUTHORS PR-URL: https://github.com/nodejs/node/pull/43872 Reviewed-By: Luigi Pinca Reviewed-By: Rich Trott Reviewed-By: Antoine du Hamel --- .mailmap | 5 +++-- AUTHORS | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index 6248917e68515d..4f14853f5be35e 100644 --- a/.mailmap +++ b/.mailmap @@ -221,9 +221,10 @@ James Beavers James Bromwell <943160+thw0rted@users.noreply.github.com> James Hartig James Ide +James Ide James M Snell James Nimlos -James Sumners +James Sumners Jan Krems Jem Bezooyen Jem Bezooyen @@ -241,7 +242,7 @@ Jesús Leganés-Combarro 'piranna Jimb Esser Jithil P Ponnan Jithil P Ponnan -Jochen Eisinger +Jochen Eisinger Joe Shaw Johan Bergström Johan Dahlberg diff --git a/AUTHORS b/AUTHORS index 029eb68e36640f..1cc18b7775cb7a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3481,5 +3481,10 @@ Alena Khineika Basit <1305718+mabaasit@users.noreply.github.com> Kid <44045911+kidonng@users.noreply.github.com> Jeremy Rose +Facundo Tuesca +Dan Castillo +Varun Sharma +txxnano +Jannis R # Generated by tools/update-authors.mjs From 822b9c3624d697c850b221658baead7bc84bc554 Mon Sep 17 00:00:00 2001 From: John Gee Date: Tue, 19 Jul 2022 05:36:30 +1200 Subject: [PATCH 04/46] util: add tokens to parseArgs Offer additional meta-data for building custom and additional behaviour on top of parseArgs. PR-URL: https://github.com/nodejs/node/pull/43459 Reviewed-By: Ben Coe Reviewed-By: Antoine du Hamel --- doc/api/util.md | 123 +++++++++- lib/internal/util/parse_args/parse_args.js | 251 ++++++++++++--------- test/parallel/test-parse-args.mjs | 245 ++++++++++++++++++++ 3 files changed, 515 insertions(+), 104 deletions(-) diff --git a/doc/api/util.md b/doc/api/util.md index 475c4aa5685ba5..c44642253a0564 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -1018,6 +1018,11 @@ equality. > Stability: 1 - Experimental @@ -1034,18 +1039,24 @@ added: REPLACEME times. If `true`, all values will be collected in an array. If `false`, values for the option are last-wins. **Default:** `false`. * `short` {string} A single character alias for the option. - * `strict`: {boolean} Should an error be thrown when unknown arguments + * `strict` {boolean} Should an error be thrown when unknown arguments are encountered, or when arguments are passed that do not match the `type` configured in `options`. **Default:** `true`. - * `allowPositionals`: {boolean} Whether this command accepts positional + * `allowPositionals` {boolean} Whether this command accepts positional arguments. **Default:** `false` if `strict` is `true`, otherwise `true`. + * `tokens` {boolean} Return the parsed tokens. This is useful for extending + the built-in behavior, from adding additional checks through to reprocessing + the tokens in different ways. + **Default:** `false`. * Returns: {Object} The parsed command line arguments: * `values` {Object} A mapping of parsed option names with their {string} or {boolean} values. * `positionals` {string\[]} Positional arguments. + * `tokens` {Object\[] | undefined} See [parseArgs tokens](#parseargs-tokens) + section. Only returned if `config` includes `tokens: true`. Provides a higher level API for command-line argument parsing than interacting with `process.argv` directly. Takes a specification for the expected arguments @@ -1094,6 +1105,114 @@ console.log(values, positionals); `util.parseArgs` is experimental and behavior may change. Join the conversation in [pkgjs/parseargs][] to contribute to the design. +### `parseArgs` `tokens` + +Detailed parse information is available for adding custom behaviours by +specifying `tokens: true` in the configuration. +The returned tokens have properties describing: + +* all tokens + * `kind` {string} One of 'option', 'positional', or 'option-terminator'. + * `index` {number} Index of element in `args` containing token. So the + source argument for a token is `args[token.index]`. +* option tokens + * `name` {string} Long name of option. + * `rawName` {string} How option used in args, like `-f` of `--foo`. + * `value` {string | undefined} Option value specified in args. + Undefined for boolean options. + * `inlineValue` {boolean | undefined} Whether option value specified inline, + like `--foo=bar`. +* positional tokens + * `value` {string} The value of the positional argument in args (i.e. `args[index]`). +* option-terminator token + +The returned tokens are in the order encountered in the input args. Options +that appear more than once in args produce a token for each use. Short option +groups like `-xy` expand to a token for each option. So `-xxx` produces +three tokens. + +For example to use the returned tokens to add support for a negated option +like `--no-color`, the tokens can be reprocessed to change the value stored +for the negated option. + +```mjs +import { parseArgs } from 'node:util'; + +const options = { + 'color': { type: 'boolean' }, + 'no-color': { type: 'boolean' }, + 'logfile': { type: 'string' }, + 'no-logfile': { type: 'boolean' }, +}; +const { values, tokens } = parseArgs({ options, tokens: true }); + +// Reprocess the option tokens and overwrite the returned values. +tokens + .filter((token) => token.kind === 'option') + .forEach((token) => { + if (token.name.startsWith('no-')) { + // Store foo:false for --no-foo + const positiveName = token.name.slice(3); + values[positiveName] = false; + delete values[token.name]; + } else { + // Resave value so last one wins if both --foo and --no-foo. + values[token.name] = token.value ?? true; + } + }); + +const color = values.color; +const logfile = values.logfile ?? 'default.log'; + +console.log({ logfile, color }); +``` + +```cjs +const { parseArgs } = require('node:util'); + +const options = { + 'color': { type: 'boolean' }, + 'no-color': { type: 'boolean' }, + 'logfile': { type: 'string' }, + 'no-logfile': { type: 'boolean' }, +}; +const { values, tokens } = parseArgs({ options, tokens: true }); + +// Reprocess the option tokens and overwrite the returned values. +tokens + .filter((token) => token.kind === 'option') + .forEach((token) => { + if (token.name.startsWith('no-')) { + // Store foo:false for --no-foo + const positiveName = token.name.slice(3); + values[positiveName] = false; + delete values[token.name]; + } else { + // Resave value so last one wins if both --foo and --no-foo. + values[token.name] = token.value ?? true; + } + }); + +const color = values.color; +const logfile = values.logfile ?? 'default.log'; + +console.log({ logfile, color }); +``` + +Example usage showing negated options, and when an option is used +multiple ways then last one wins. + +```console +$ node negate.js +{ logfile: 'default.log', color: undefined } +$ node negate.js --no-logfile --no-color +{ logfile: false, color: false } +$ node negate.js --logfile=test.log --color +{ logfile: 'test.log', color: true } +$ node negate.js --no-logfile --logfile=test.log --color --no-color +{ logfile: 'test.log', color: false } +``` + ## `util.promisify(original)` + +* `request` {http.IncomingMessage} Arguments for the HTTP request, as it is in + the [`'request'`][] event +* `socket` {stream.Duplex} Network socket between the server and client + +When the number of requests on a socket reaches the threshold of +`server.maxRequestsPerSocket`, the server will drop new requests +and emit `'dropRequest'` event instead, then send `503` to client. + ### Event: `'request'` -* {stream.Readable} +* {stream.Readable|null|undefined} A `Readable Stream` that represents the child process's `stderr`. @@ -1649,8 +1649,8 @@ then this will be `null`. `subprocess.stderr` is an alias for `subprocess.stdio[2]`. Both properties will refer to the same value. -The `subprocess.stderr` property can be `null` if the child process could -not be successfully spawned. +The `subprocess.stderr` property can be `null` or `undefined` +if the child process could not be successfully spawned. ### `subprocess.stdin` @@ -1658,7 +1658,7 @@ not be successfully spawned. added: v0.1.90 --> -* {stream.Writable} +* {stream.Writable|null|undefined} A `Writable Stream` that represents the child process's `stdin`. @@ -1671,8 +1671,8 @@ then this will be `null`. `subprocess.stdin` is an alias for `subprocess.stdio[0]`. Both properties will refer to the same value. -The `subprocess.stdin` property can be `undefined` if the child process could -not be successfully spawned. +The `subprocess.stdin` property can be `null` or `undefined` +if the child process could not be successfully spawned. ### `subprocess.stdio` @@ -1724,7 +1724,7 @@ not be successfully spawned. added: v0.1.90 --> -* {stream.Readable} +* {stream.Readable|null|undefined} A `Readable Stream` that represents the child process's `stdout`. @@ -1744,8 +1744,8 @@ subprocess.stdout.on('data', (data) => { }); ``` -The `subprocess.stdout` property can be `null` if the child process could -not be successfully spawned. +The `subprocess.stdout` property can be `null` or `undefined` +if the child process could not be successfully spawned. ### `subprocess.unref()` From 8339409ac621cefeef1053b812cf02423bd7b65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Nie=C3=9Fen?= Date: Sun, 24 Jul 2022 18:56:40 +0200 Subject: [PATCH 33/46] test: simplify test-tls-set-secure-context Instead of recursively scheduling makeRemainingRequests and ignoring its outcome, safely loop within the async function. Avoid unncessary async lambda functions passed to assert.rejects. Use events.once() to simplify control flow in makeRequest, instead of manually constructing a Promise. PR-URL: https://github.com/nodejs/node/pull/43878 Reviewed-By: Antoine du Hamel --- test/parallel/test-tls-set-secure-context.js | 69 ++++++++------------ 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/test/parallel/test-tls-set-secure-context.js b/test/parallel/test-tls-set-secure-context.js index d62b6b9f94f601..c056875e14ddfb 100644 --- a/test/parallel/test-tls-set-secure-context.js +++ b/test/parallel/test-tls-set-secure-context.js @@ -9,7 +9,9 @@ if (!common.hasCrypto) // secure context is changed. const assert = require('assert'); +const events = require('events'); const https = require('https'); +const timers = require('timers/promises'); const fixtures = require('../common/fixtures'); const credentialOptions = [ { @@ -43,10 +45,10 @@ server.listen(0, common.mustCall(() => { const { port } = server.address(); const firstRequest = makeRequest(port, 1); - async function makeRemainingRequests() { + (async function makeRemainingRequests() { // Wait until the first request is guaranteed to have been handled. - if (!firstResponse) { - return setImmediate(makeRemainingRequests); + while (!firstResponse) { + await timers.setImmediate(); } assert.strictEqual(await makeRequest(port, 2), 'success'); @@ -56,53 +58,38 @@ server.listen(0, common.mustCall(() => { const errorMessageRegex = common.hasOpenSSL3 ? /^Error: self-signed certificate$/ : /^Error: self signed certificate$/; - await assert.rejects(async () => { - await makeRequest(port, 3); - }, errorMessageRegex); + await assert.rejects(makeRequest(port, 3), errorMessageRegex); server.setSecureContext(credentialOptions[0]); assert.strictEqual(await makeRequest(port, 4), 'success'); server.setSecureContext(credentialOptions[1]); firstResponse.end('fun!'); - await assert.rejects(async () => { - await makeRequest(port, 5); - }, errorMessageRegex); + await assert.rejects(makeRequest(port, 5), errorMessageRegex); assert.strictEqual(await firstRequest, 'multi-request-success-fun!'); server.close(); - } - - makeRemainingRequests(); + })().then(common.mustCall()); })); -function makeRequest(port, id) { - return new Promise((resolve, reject) => { - const options = { - rejectUnauthorized: true, - ca: credentialOptions[0].ca, - servername: 'agent1', - headers: { id } - }; - - let errored = false; - https.get(`https://localhost:${port}`, options, (res) => { - let response = ''; - - res.setEncoding('utf8'); - - res.on('data', (chunk) => { - response += chunk; - }); - - res.on('end', common.mustCall(() => { - resolve(response); - })); - }).on('error', (err) => { - errored = true; - reject(err); - }).on('finish', () => { - assert.strictEqual(errored, false); - }); - }); +async function makeRequest(port, id) { + const options = { + rejectUnauthorized: true, + ca: credentialOptions[0].ca, + servername: 'agent1', + headers: { id }, + agent: new https.Agent() + }; + + const req = https.get(`https://localhost:${port}`, options); + + let errored = false; + req.on('error', () => errored = true); + req.on('finish', () => assert.strictEqual(errored, false)); + + const [res] = await events.once(req, 'response'); + res.setEncoding('utf8'); + let response = ''; + for await (const chunk of res) response += chunk; + return response; } From 75e60cb81cbcc88084e672d1a0d44e289039d2b4 Mon Sep 17 00:00:00 2001 From: Andreu Botella Date: Mon, 25 Jul 2022 00:59:25 +0200 Subject: [PATCH 34/46] test: work scheduled in process.nextTick can keep the event loop alive PR-URL: https://github.com/nodejs/node/pull/43787 Reviewed-By: Antoine du Hamel Reviewed-By: Ujjwal Sharma --- test/parallel/test-process-beforeexit.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/parallel/test-process-beforeexit.js b/test/parallel/test-process-beforeexit.js index 7b7ba48f7d8cb4..e04b756cade8bc 100644 --- a/test/parallel/test-process-beforeexit.js +++ b/test/parallel/test-process-beforeexit.js @@ -59,12 +59,21 @@ function tryRepeatedTimer() { if (++n < N) setTimeout(repeatedTimer, 1); else // n == N - process.once('beforeExit', common.mustCall(tryNextTick)); + process.once('beforeExit', common.mustCall(tryNextTickSetImmediate)); }, N); setTimeout(repeatedTimer, 1); } // Test if the callback of `process.nextTick` can be invoked. +function tryNextTickSetImmediate() { + process.nextTick(common.mustCall(function() { + setImmediate(common.mustCall(() => { + process.once('beforeExit', common.mustCall(tryNextTick)); + })); + })); +} + +// Test that `process.nextTick` won't keep the event loop running by itself. function tryNextTick() { process.nextTick(common.mustCall(function() { process.once('beforeExit', common.mustNotCall()); From 105453b34521886d051fb8dd92af2db82b7a4a89 Mon Sep 17 00:00:00 2001 From: Keyhan Vakil Date: Sun, 24 Jul 2022 16:06:08 -0700 Subject: [PATCH 35/46] buffer: do not leak memory if buffer is too big MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recent pull request changed this method to throw when the buffer was too big, but this meant that the `free` finalizer would never get called, leading to a memory leak. A previous version of this diff included a test provoking this behavior with `v8.serialize`, but it unfortunately kept triggering the OOM killer, so it was removed. Refs: https://github.com/nodejs/node/pull/40243 PR-URL: https://github.com/nodejs/node/pull/43938 Reviewed-By: Darshan Sen Reviewed-By: Tobias Nießen Reviewed-By: Ben Noordhuis --- src/node_buffer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/node_buffer.cc b/src/node_buffer.cc index 5b2186feb8c707..aec97f15e2c809 100644 --- a/src/node_buffer.cc +++ b/src/node_buffer.cc @@ -497,6 +497,7 @@ MaybeLocal New(Environment* env, if (length > kMaxLength) { Isolate* isolate(env->isolate()); isolate->ThrowException(ERR_BUFFER_TOO_LARGE(isolate)); + free(data); return Local(); } } From 7acbe8f53f7833fc144d25597b37f8b2c96d4c00 Mon Sep 17 00:00:00 2001 From: theanarkh Date: Mon, 25 Jul 2022 07:06:20 +0800 Subject: [PATCH 36/46] trace_events: trace net connect event PR-URL: https://github.com/nodejs/node/pull/43903 Reviewed-By: Matteo Collina Reviewed-By: Chengzhong Wu Reviewed-By: Darshan Sen Reviewed-By: Minwoo Jung --- doc/api/tracing.md | 1 + src/connection_wrap.cc | 6 ++++ src/pipe_wrap.cc | 6 ++++ src/tcp_wrap.cc | 12 ++++++- test/parallel/test-trace-events-net.js | 45 ++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 test/parallel/test-trace-events-net.js diff --git a/doc/api/tracing.md b/doc/api/tracing.md index 890938eb9e1396..7766412ed0b75a 100644 --- a/doc/api/tracing.md +++ b/doc/api/tracing.md @@ -23,6 +23,7 @@ The available categories are: * `node.console`: Enables capture of `console.time()` and `console.count()` output. * `node.dns.native`: Enables capture of trace data for DNS queries. +* `node.net.native`: Enables capture of trace data for network. * `node.environment`: Enables capture of Node.js Environment milestones. * `node.fs.sync`: Enables capture of trace data for file system sync methods. * `node.perf`: Enables capture of [Performance API][] measurements. diff --git a/src/connection_wrap.cc b/src/connection_wrap.cc index 300f7d316b8309..29815eee4381a0 100644 --- a/src/connection_wrap.cc +++ b/src/connection_wrap.cc @@ -108,6 +108,12 @@ void ConnectionWrap::AfterConnect(uv_connect_t* req, Boolean::New(env->isolate(), writable) }; + TRACE_EVENT_NESTABLE_ASYNC_END1(TRACING_CATEGORY_NODE2(net, native), + "connect", + req_wrap.get(), + "status", + status); + req_wrap->MakeCallback(env->oncomplete_string(), arraysize(argv), argv); } diff --git a/src/pipe_wrap.cc b/src/pipe_wrap.cc index da52f5cee01062..85daf4a1e618ae 100644 --- a/src/pipe_wrap.cc +++ b/src/pipe_wrap.cc @@ -241,6 +241,12 @@ void PipeWrap::Connect(const FunctionCallbackInfo& args) { *name, AfterConnect); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN1(TRACING_CATEGORY_NODE2(net, native), + "connect", + req_wrap, + "pipe_path", + TRACE_STR_COPY(*name)); + args.GetReturnValue().Set(0); // uv_pipe_connect() doesn't return errors. } diff --git a/src/tcp_wrap.cc b/src/tcp_wrap.cc index 2e6c884c63f937..0fcc12d649eac4 100644 --- a/src/tcp_wrap.cc +++ b/src/tcp_wrap.cc @@ -335,8 +335,18 @@ void TCPWrap::Connect(const FunctionCallbackInfo& args, &wrap->handle_, reinterpret_cast(&addr), AfterConnect); - if (err) + if (err) { delete req_wrap; + } else { + int port = args[2]->Uint32Value(env->context()).FromJust(); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN2(TRACING_CATEGORY_NODE2(net, native), + "connect", + req_wrap, + "ip", + TRACE_STR_COPY(*ip_address), + "port", + port); + } } args.GetReturnValue().Set(err); diff --git a/test/parallel/test-trace-events-net.js b/test/parallel/test-trace-events-net.js new file mode 100644 index 00000000000000..227c168c013245 --- /dev/null +++ b/test/parallel/test-trace-events-net.js @@ -0,0 +1,45 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const cp = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const tmpdir = require('../common/tmpdir'); + +const CODE = ` + const net = require('net'); + const socket = net.connect('${common.PIPE}'); + socket.on('error', () => {}); + const server = net.createServer((socket) => { + socket.destroy(); + server.close(); + }).listen(0, () => { + net.connect(server.address().port); + }); +`; + +tmpdir.refresh(); +const FILE_NAME = path.join(tmpdir.path, 'node_trace.1.log'); + +const proc = cp.spawn(process.execPath, + [ '--trace-events-enabled', + '--trace-event-categories', 'node.net.native', + '-e', CODE ], + { cwd: tmpdir.path }); + +proc.once('exit', common.mustCall(() => { + assert(fs.existsSync(FILE_NAME)); + fs.readFile(FILE_NAME, common.mustCall((err, data) => { + const traces = JSON.parse(data.toString()).traceEvents; + assert(traces.length > 0); + let count = 0; + traces.forEach((trace) => { + if (trace.cat === 'node,node.net,node.net.native' && + trace.name === 'connect') { + count++; + } + }); + // Two begin, two end + assert.strictEqual(count, 4); + })); +})); From 5e9bdb668385170865dd58977b23db1fec616ab0 Mon Sep 17 00:00:00 2001 From: Feng Yu Date: Mon, 25 Jul 2022 07:06:32 +0800 Subject: [PATCH 37/46] test: remove test-whatwg-events-add-event-listener-options-once.js PR-URL: https://github.com/nodejs/node/pull/43877 Reviewed-By: Antoine du Hamel --- ...-events-add-event-listener-options-once.js | 132 ------------------ 1 file changed, 132 deletions(-) delete mode 100644 test/parallel/test-whatwg-events-add-event-listener-options-once.js diff --git a/test/parallel/test-whatwg-events-add-event-listener-options-once.js b/test/parallel/test-whatwg-events-add-event-listener-options-once.js deleted file mode 100644 index 9c74829d761306..00000000000000 --- a/test/parallel/test-whatwg-events-add-event-listener-options-once.js +++ /dev/null @@ -1,132 +0,0 @@ -'use strict'; - -const common = require('../common'); - -const { - strictEqual, -} = require('assert'); - -// Manually ported from: wpt@dom/events/AddEventListenerOptions-once.html -{ - const document = new EventTarget(); - let invoked_once = false; - let invoked_normal = false; - function handler_once() { - invoked_once = true; - } - - function handler_normal() { - invoked_normal = true; - } - - document.addEventListener('test', handler_once, { once: true }); - document.addEventListener('test', handler_normal); - document.dispatchEvent(new Event('test')); - strictEqual(invoked_once, true, 'Once handler should be invoked'); - strictEqual(invoked_normal, true, 'Normal handler should be invoked'); - - invoked_once = false; - invoked_normal = false; - document.dispatchEvent(new Event('test')); - strictEqual(invoked_once, false, 'Once handler shouldn\'t be invoked again'); - strictEqual(invoked_normal, true, 'Normal handler should be invoked again'); - document.removeEventListener('test', handler_normal); -} - - -{ - // Manually ported from AddEventListenerOptions-once.html - const document = new EventTarget(); - let invoked_count = 0; - function handler() { - invoked_count++; - } - document.addEventListener('test', handler, { once: true }); - document.addEventListener('test', handler); - document.dispatchEvent(new Event('test')); - strictEqual(invoked_count, 1, 'The handler should only be added once'); - - invoked_count = 0; - document.dispatchEvent(new Event('test')); - strictEqual(invoked_count, 0, 'The handler was added as a once listener'); - - invoked_count = 0; - document.addEventListener('test', handler, { once: true }); - document.removeEventListener('test', handler); - document.dispatchEvent(new Event('test')); - strictEqual(invoked_count, 0, 'The handler should have been removed'); -} - -{ - // TODO(benjamingr) fix EventTarget recursion - common.skip('EventTarget recursion is currently broken'); - const document = new EventTarget(); - let invoked_count = 0; - function handler() { - invoked_count++; - if (invoked_count === 1) - document.dispatchEvent(new Event('test')); - } - document.addEventListener('test', handler, { once: true }); - document.dispatchEvent(new Event('test')); - strictEqual(invoked_count, 1, 'Once handler should only be invoked once'); - - invoked_count = 0; - function handler2() { - invoked_count++; - if (invoked_count === 1) - document.addEventListener('test', handler2, { once: true }); - if (invoked_count <= 2) - document.dispatchEvent(new Event('test')); - } - document.addEventListener('test', handler2, { once: true }); - document.dispatchEvent(new Event('test')); - strictEqual(invoked_count, 2, 'Once handler should only be invoked once'); -} - -// Manually converted from https://github.com/web-platform-tests/wpt/blob/master/dom/events/AddEventListenerOptions-once.html -// in order to define the `document` ourselves - -{ - const document = new EventTarget(); - - // Should only fire for first event - document.addEventListener('test', common.mustCall(1), { once: true }); - // Should fire for both events - document.addEventListener('test', common.mustCall(2)); - // Fire events - document.dispatchEvent(new Event('test')); - document.dispatchEvent(new Event('test')); -} -{ - const document = new EventTarget(); - - const handler = common.mustCall(2); - // Both should only fire on first event - document.addEventListener('test', handler.bind(), { once: true }); - document.addEventListener('test', handler.bind(), { once: true }); - // Fire events - document.dispatchEvent(new Event('test')); - document.dispatchEvent(new Event('test')); -} -{ - const document = new EventTarget(); - - const handler = common.mustCall(2); - - // Should only fire once on first event - document.addEventListener('test', common.mustCall(1), { once: true }); - // Should fire twice until removed - document.addEventListener('test', handler); - // Fire two events - document.dispatchEvent(new Event('test')); - document.dispatchEvent(new Event('test')); - - // Should only fire once on the next event - document.addEventListener('test', common.mustCall(1), { once: true }); - // The previous handler should no longer fire - document.removeEventListener('test', handler); - - // Fire final event triggering - document.dispatchEvent(new Event('test')); -} From e6048b01232e76f8b976f69b1eb20ba4e031c1a2 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 25 Jul 2022 06:37:17 +0200 Subject: [PATCH 38/46] inspector: set sampling interval before start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/43779 Reviewed-By: Joyee Cheung Reviewed-By: Colin Ihrig Reviewed-By: Michaël Zasso Reviewed-By: Darshan Sen --- src/inspector_profiler.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inspector_profiler.cc b/src/inspector_profiler.cc index db8bda8524d3b4..332aa536364fba 100644 --- a/src/inspector_profiler.cc +++ b/src/inspector_profiler.cc @@ -331,11 +331,11 @@ MaybeLocal V8CpuProfilerConnection::GetProfile(Local result) { void V8CpuProfilerConnection::Start() { DispatchMessage("Profiler.enable"); - DispatchMessage("Profiler.start"); std::string params = R"({ "interval": )"; params += std::to_string(env()->cpu_prof_interval()); params += " }"; DispatchMessage("Profiler.setSamplingInterval", params.c_str()); + DispatchMessage("Profiler.start"); } void V8CpuProfilerConnection::End() { From b957882826b85a505039c744f8ab1862a59c92b3 Mon Sep 17 00:00:00 2001 From: Daeyeon Jeong Date: Mon, 25 Jul 2022 17:09:49 +0900 Subject: [PATCH 39/46] net: fix socket._getpeername Fixes: https://github.com/nodejs/node/issues/43009 If calling `this._handle.getpeername` returns an error at the first call, its result shouldn't be cached to `this._peername`. Signed-off-by: Daeyeon Jeong daeyeon.dev@gmail.com PR-URL: https://github.com/nodejs/node/pull/43010 Reviewed-By: Paolo Insogna Reviewed-By: Matteo Collina Reviewed-By: Zeyu "Alex" Yang --- lib/net.js | 7 ++++--- test/parallel/test-net-remote-address.js | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 test/parallel/test-net-remote-address.js diff --git a/lib/net.js b/lib/net.js index 9670a662a996ea..e3d490ceed8ed9 100644 --- a/lib/net.js +++ b/lib/net.js @@ -785,9 +785,10 @@ Socket.prototype._getpeername = function() { if (!this._handle || !this._handle.getpeername) { return this._peername || {}; } else if (!this._peername) { - this._peername = {}; - // FIXME(bnoordhuis) Throw when the return value is not 0? - this._handle.getpeername(this._peername); + const out = {}; + const err = this._handle.getpeername(out); + if (err) return out; + this._peername = out; } return this._peername; }; diff --git a/test/parallel/test-net-remote-address.js b/test/parallel/test-net-remote-address.js new file mode 100644 index 00000000000000..a116cb99d3bcab --- /dev/null +++ b/test/parallel/test-net-remote-address.js @@ -0,0 +1,23 @@ +'use strict'; + +const common = require('../common'); +const net = require('net'); +const { strictEqual } = require('assert'); + +const server = net.createServer(); + +server.listen(common.mustCall(function() { + const socket = net.connect({ port: server.address().port }); + + strictEqual(socket.connecting, true); + strictEqual(socket.remoteAddress, undefined); + + socket.on('connect', common.mustCall(function() { + strictEqual(socket.remoteAddress !== undefined, true); + socket.end(); + })); + + socket.on('end', common.mustCall(function() { + server.close(); + })); +})); From 24b4a59fd1e7c738e501340615ec7de2090f8b07 Mon Sep 17 00:00:00 2001 From: Keyhan Vakil Date: Mon, 25 Jul 2022 04:31:42 -0700 Subject: [PATCH 40/46] doc: recommend git-node-v8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the comments in #43924, almost everyone uses `git-node-v8`. I included example steps for using `git-node-v8`. I ran through both of these instructions on a clean Linux machine (I had to fudge the patch SHA of course) and they seemed to work. Refs: https://github.com/nodejs/node/pull/43924 PR-URL: https://github.com/nodejs/node/pull/43934 Reviewed-By: Antoine du Hamel Reviewed-By: Michaël Zasso Reviewed-By: Richard Lau --- doc/contributing/maintaining-V8.md | 84 ++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/doc/contributing/maintaining-V8.md b/doc/contributing/maintaining-V8.md index 83fad7000f1bf3..0143cf1c7720f5 100644 --- a/doc/contributing/maintaining-V8.md +++ b/doc/contributing/maintaining-V8.md @@ -208,6 +208,64 @@ backport the fix: Abandoned V8 branches are supported in the Node.js repository. The fix needs to be cherry-picked in the Node.js repository and V8-CI must test the change. +As an example for how to backport changes, consider the bug +[RegExp show inconsistent result with other browsers](https://crbug.com/v8/5199). +From the bug we can see that it was merged by V8 into 5.2 and 5.3, and not into +V8 5.1 (since it was already abandoned). Since Node.js `v6.x` uses V8 5.1, the +fix needed to be backported. + +#### Backporting with `git-node` (recommended) + +You can use [`git-node`][] to help you backport patches. This removes +some manual steps and is recommended. + +Here are the steps for the bug mentioned above: + +1. Install `git-node` by installing [`node-core-utils`][]. +2. Install the prerequisites for [`git-node-v8`][]. +3. Find the commit hash linked-to in the issue (in this case a51f429). +4. Checkout a branch off the appropriate _vY.x-staging_ branch (e.g. + _v6.x-staging_ to fix an issue in V8 5.1). +5. Run `git node v8 backport a51f429`. +6. If there are conflicts, `git-node` will wait for you to resolve them: + +```console +$ git node v8 backport a51f429 +✔ Update local V8 clone +❯ V8 commit backport + ✔ Get current V8 version + ✔ Generate patches + ❯ Apply and commit patches to deps/v8 + ❯ Commit a51f429772d1 + ⠏ Apply patch + ◼ Increment embedder version number + ◼ Commit patch + +? Resolve merge conflicts and enter 'RESOLVED' ‣ +``` + +Resolve conflicts, stage the files (you may need to open another terminal or use +a GUI git client), then return to the terminal running `git-node`, type +`RESOLVED`, and hit Enter. + +7. After you resolve conflicts (or if there are no conflicts), the + output should look like this: + +```console +$ git node v8 backport a51f429 +✔ Update local V8 clone +✔ V8 commit backport +``` + +8. Open a PR against the v6.x-staging branch in the Node.js repository. + Launch the normal and [V8 CI][] using the Node.js CI system. We only + needed to backport to v6.x as the other LTS branches weren't affected + by this bug. + +See [`git-node-v8-backport`][] for more documentation and additional options. + +#### Backporting manually + * For each abandoned V8 branch corresponding to an LTS branch that is affected by the bug: * Checkout a branch off the appropriate _vY.x-staging_ branch (e.g. @@ -224,14 +282,7 @@ to be cherry-picked in the Node.js repository and V8-CI must test the change. `tools/make-v8.sh` to reconstruct a git tree in the `deps/v8` directory to run V8 tests.[^2] -The [`git-node`][] tool can be used to simplify this task. Run -`git node v8 backport ` to cherry-pick a commit. - -An example for workflow how to cherry-pick consider the bug -[RegExp show inconsistent result with other browsers](https://crbug.com/v8/5199). -From the bug we can see that it was merged by V8 into 5.2 and 5.3, and not into -V8 5.1 (since it was already abandoned). Since Node.js `v6.x` uses V8 5.1, the -fix needed to be cherry-picked. To cherry-pick, here's an example workflow: +Here are the steps for the bug mentioned above: * Download and apply the commit linked-to in the issue (in this case a51f429): @@ -323,6 +374,16 @@ compute the diff between these tags on the V8 repository, and then apply that patch on the copy of V8 in Node.js. This should preserve the patches/backports that Node.js may be floating (or else cause a merge conflict). +#### Applying minor updates with `git-node` (recommended) + +1. Install [`git-node`][] by installing [`node-core-utils`][]. +2. Install the prerequisites for [`git-node-v8`][]. +3. Run `git node v8 minor` to apply a minor update. + +See [`git-node-v8-minor`][] for more documentation and additional options. + +#### Applying minor updates manually + The rough outline of the process is: ```bash @@ -341,9 +402,6 @@ curl -L https://github.com/v8/v8/compare/${V8_OLD_VERSION}...${V8_NEW_VERSION}.p V8 also keeps tags of the form _5.4-lkgr_ which point to the _Last Known Good Revision_ from the 5.4 branch that can be useful in the update process above. -The [`git-node`][] tool can be used to simplify this task. Run `git node v8 minor` -to apply a minor update. - ### Major updates We upgrade the version of V8 in Node.js `main` whenever a V8 release goes stable @@ -430,4 +488,8 @@ This would require some tooling to: [V8MergingPatching]: https://v8.dev/docs/merge-patch [V8TemplateMergeRequest]: https://bugs.chromium.org/p/v8/issues/entry?template=Node.js%20merge%20request [V8TemplateUpstreamBug]: https://bugs.chromium.org/p/v8/issues/entry?template=Node.js%20upstream%20bug +[`git-node-v8-backport`]: https://github.com/nodejs/node-core-utils/blob/main/docs/git-node.md#git-node-v8-backport-sha +[`git-node-v8-minor`]: https://github.com/nodejs/node-core-utils/blob/main/docs/git-node.md#git-node-v8-minor +[`git-node-v8`]: https://github.com/nodejs/node-core-utils/blob/HEAD/docs/git-node.md#git-node-v8 [`git-node`]: https://github.com/nodejs/node-core-utils/blob/HEAD/docs/git-node.md#git-node-v8 +[`node-core-utils`]: https://github.com/nodejs/node-core-utils#Install From 8744e6b23b0035f889849136ff71ffa132ab9024 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Mon, 25 Jul 2022 14:01:41 +0200 Subject: [PATCH 41/46] tools: add more options to track flaky tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs: https://github.com/nodejs/node/pull/43929#issuecomment-1193104729 PR-URL: https://github.com/nodejs/node/pull/43954 Reviewed-By: Matteo Collina Reviewed-By: Tobias Nießen Reviewed-By: Feng Yu --- .github/workflows/build-tarball.yml | 4 +-- .github/workflows/build-windows.yml | 2 +- .github/workflows/coverage-linux.yml | 4 +-- .github/workflows/coverage-windows.yml | 2 +- .github/workflows/doc.yml | 2 +- .github/workflows/test-asan.yml | 4 +-- .github/workflows/test-internet.yml | 2 +- .github/workflows/test-linux.yml | 4 +-- .github/workflows/test-macos.yml | 4 +-- tools/test.py | 41 ++++++++++++++++++-------- 10 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-tarball.yml b/.github/workflows/build-tarball.yml index 4509450172f277..2ced884f35c539 100644 --- a/.github/workflows/build-tarball.yml +++ b/.github/workflows/build-tarball.yml @@ -29,7 +29,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read @@ -94,4 +94,4 @@ jobs: - name: Test run: | cd $TAR_DIR - make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" + make run-ci -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9" diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 3e7ec807a34799..9b5f81a816e2d0 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -24,7 +24,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index abd69801f60045..db0a82c9e21d73 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -27,7 +27,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read @@ -53,7 +53,7 @@ jobs: # TODO(bcoe): fix the couple tests that fail with the inspector enabled. # The cause is most likely coverage's use of the inspector. - name: Test - run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 + run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9" || exit 0 - name: Report JS run: npx c8 report --check-coverage env: diff --git a/.github/workflows/coverage-windows.yml b/.github/workflows/coverage-windows.yml index d9c5bfb58d61e6..78465acb17566d 100644 --- a/.github/workflows/coverage-windows.yml +++ b/.github/workflows/coverage-windows.yml @@ -29,7 +29,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 72abb16ad5083c..76660343ca2f46 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -40,4 +40,4 @@ jobs: name: docs path: out/doc - name: Test - run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions" + run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions --measure-flakiness 9" diff --git a/.github/workflows/test-asan.yml b/.github/workflows/test-asan.yml index ba30449e90ba40..c8e7a09e2e4efd 100644 --- a/.github/workflows/test-asan.yml +++ b/.github/workflows/test-asan.yml @@ -31,7 +31,7 @@ concurrency: env: ASAN_OPTIONS: intercept_tls_get_addr=0 PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read @@ -58,4 +58,4 @@ jobs: - name: Build run: make build-ci -j2 V=1 - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300 --measure-flakiness 9" diff --git a/.github/workflows/test-internet.yml b/.github/workflows/test-internet.yml index 297ceea987a76a..4fcb18a14f5ce9 100644 --- a/.github/workflows/test-internet.yml +++ b/.github/workflows/test-internet.yml @@ -22,7 +22,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 4cc09f22c25105..24196849d1bac8 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -24,7 +24,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read @@ -46,4 +46,4 @@ jobs: - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn" - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9" diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 5f93730d666fc0..7faddc8eaedaec 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -30,7 +30,7 @@ concurrency: env: PYTHON_VERSION: '3.10' - FLAKY_TESTS: dontcare + FLAKY_TESTS: keep_retrying permissions: contents: read @@ -60,4 +60,4 @@ jobs: - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn" - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9" diff --git a/tools/test.py b/tools/test.py index 9a7de5ed24d706..35153057f0b9f2 100755 --- a/tools/test.py +++ b/tools/test.py @@ -96,10 +96,11 @@ def get_module(name, path): class ProgressIndicator(object): - def __init__(self, cases, flaky_tests_mode): + def __init__(self, cases, flaky_tests_mode, measure_flakiness): self.cases = cases self.serial_id = 0 self.flaky_tests_mode = flaky_tests_mode + self.measure_flakiness = measure_flakiness self.parallel_queue = Queue(len(cases)) self.sequential_queue = Queue(len(cases)) for case in cases: @@ -211,10 +212,22 @@ def RunSingle(self, parallel, thread_id): if output.UnexpectedOutput(): if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE: self.flaky_failed.append(output) + elif FLAKY in output.test.outcomes and self.flaky_tests_mode == KEEP_RETRYING: + for _ in range(99): + if not case.Run().UnexpectedOutput(): + self.flaky_failed.append(output) + break + else: + # If after 100 tries, the test is not passing, it's not flaky. + self.failed.append(output) else: self.failed.append(output) if output.HasCrashed(): self.crashed += 1 + if self.measure_flakiness: + outputs = [case.Run() for _ in range(self.measure_flakiness)] + # +1s are there because the test already failed once at this point. + print(f" failed {len([i for i in outputs if i.UnexpectedOutput()]) + 1} out of {self.measure_flakiness + 1}") else: self.succeeded += 1 self.remaining -= 1 @@ -436,8 +449,8 @@ def Done(self): class CompactProgressIndicator(ProgressIndicator): - def __init__(self, cases, flaky_tests_mode, templates): - super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode) + def __init__(self, cases, flaky_tests_mode, measure_flakiness, templates): + super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness) self.templates = templates self.last_status_length = 0 self.start_time = time.time() @@ -492,13 +505,13 @@ def PrintProgress(self, name): class ColorProgressIndicator(CompactProgressIndicator): - def __init__(self, cases, flaky_tests_mode): + def __init__(self, cases, flaky_tests_mode, measure_flakiness): templates = { 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", 'stdout': "\033[1m%s\033[0m", 'stderr': "\033[31m%s\033[0m", } - super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) + super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates) def ClearLine(self, last_line_length): print("\033[1K\r", end='') @@ -506,7 +519,7 @@ def ClearLine(self, last_line_length): class MonochromeProgressIndicator(CompactProgressIndicator): - def __init__(self, cases, flaky_tests_mode): + def __init__(self, cases, flaky_tests_mode, measure_flakiness): templates = { 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", 'stdout': '%s', @@ -514,7 +527,7 @@ def __init__(self, cases, flaky_tests_mode): 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), 'max_length': 78 } - super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) + super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates) def ClearLine(self, last_line_length): print(("\r" + (" " * last_line_length) + "\r"), end='') @@ -948,8 +961,8 @@ def GetTimeout(self, mode, section=''): timeout = timeout * 6 return timeout -def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode): - progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode) +def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode, measure_flakiness): + progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode, measure_flakiness) return progress.Run(tasks) # ------------------------------------------- @@ -967,6 +980,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode): SLOW = 'slow' FLAKY = 'flaky' DONTCARE = 'dontcare' +KEEP_RETRYING = 'keep_retrying' class Expression(object): pass @@ -1355,8 +1369,11 @@ def BuildOptions(): result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") result.add_option("--flaky-tests", - help="Regard tests marked as flaky (run|skip|dontcare)", + help="Regard tests marked as flaky (run|skip|dontcare|keep_retrying)", default="run") + result.add_option("--measure-flakiness", + help="When a test fails, re-run it x number of times", + default=0, type="int") result.add_option("--skip-tests", help="Tests that should not be executed (comma-separated)", default="") @@ -1433,7 +1450,7 @@ def ProcessOptions(options): # -j and ignoring -J, which is the opposite of what we used to do before -J # became a legacy no-op. print('Warning: Legacy -J option is ignored. Using the -j option.') - if options.flaky_tests not in [RUN, SKIP, DONTCARE]: + if options.flaky_tests not in [RUN, SKIP, DONTCARE, KEEP_RETRYING]: print("Unknown flaky-tests mode %s" % options.flaky_tests) return False return True @@ -1733,7 +1750,7 @@ def should_keep(case): else: try: start = time.time() - if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests): + if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests, options.measure_flakiness): result = 0 else: result = 1 From 122864fcf39dde308358ebe374766fd61f6e8ca8 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Mon, 25 Jul 2022 16:01:22 +0200 Subject: [PATCH 42/46] doc: improve documentation for safe `Promise` statics alternatives PR-URL: https://github.com/nodejs/node/pull/43759 Refs: https://github.com/nodejs/node/pull/43728 Reviewed-By: Feng Yu --- doc/contributing/primordials.md | 37 +++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/doc/contributing/primordials.md b/doc/contributing/primordials.md index 9a081418981808..8be3f8770768f9 100644 --- a/doc/contributing/primordials.md +++ b/doc/contributing/primordials.md @@ -363,10 +363,33 @@ Object.defineProperty(Object.prototype, Symbol.isConcatSpreadable, { // 1. Lookup @@iterator property on `array` (user-mutable if user-provided). // 2. Lookup @@iterator property on %Array.prototype% (user-mutable). // 3. Lookup `next` property on %ArrayIteratorPrototype% (user-mutable). -PromiseAll(array); // unsafe +PromiseAll([]); // unsafe + +PromiseAll(new SafeArrayIterator([])); // safe + +const array = [promise]; +const set = new SafeSet().add(promise); +// When running one of these functions on a non-empty iterable, it will also: +// 4. Lookup `then` property on `promise` (user-mutable if user-provided). +// 5. Lookup `then` property on `%Promise.prototype%` (user-mutable). +PromiseAll(new SafeArrayIterator(array)); // unsafe + +PromiseAll(set); // unsafe -PromiseAll(new SafeArrayIterator(array)); SafePromiseAll(array); // safe + +// Some key differences between `SafePromise[...]` and `Promise[...]` methods: + +// 1. SafePromiseAll, SafePromiseAllSettled, SafePromiseAny, and SafePromiseRace +// support passing a mapperFunction as second argument. +SafePromiseAll(ArrayPrototypeMap(array, someFunction)); +SafePromiseAll(array, someFunction); // Same as the above, but more efficient. + +// 2. SafePromiseAll, SafePromiseAllSettled, SafePromiseAny, and SafePromiseRace +// only support arrays, not iterables. Use ArrayFrom to convert an iterable +// to an array. +SafePromiseAll(set); // ignores set content. +SafePromiseAll(ArrayFrom(set)); // safe ``` @@ -459,6 +482,7 @@ original methods: * It expects an array (or array-like object) instead of an iterable. * It wraps each promise in `SafePromise` objects and wraps the result in a new `Promise` instance – which may come with a performance penalty. +* It accepts a `mapperFunction` as second argument. * Because it doesn't look up `then` property, it may not be the right tool to handle user-provided promises (which may be instances of a subclass of `Promise`). @@ -493,6 +517,15 @@ PromisePrototypeThen( process.on('exit', () => console.log(thenBlockExecuted)); // true ``` +A common pattern is to map on the array of `Promise`s to apply some +transformations, in that case it can be more efficient to pass a second argument +rather than invoking `%Array.prototype.map%`. + +```js +SafePromiseAll(ArrayPrototypeMap(array, someFunction)); +SafePromiseAll(array, someFunction); // Same as the above, but more efficient. +``` + ### (Async) Generator functions From 7f3343308fe554728bf286679e3e303b739e84ac Mon Sep 17 00:00:00 2001 From: Keyhan Vakil Date: Sat, 16 Jul 2022 18:36:31 +0000 Subject: [PATCH 43/46] doc: inspector.close undefined in worker threads In the main thread, `inspector.close` is defined as `process._debugEnd`: ``` > inspector.close [Function: _debugEnd] ``` It's not defined in worker threads: ``` > const {Worker} = require("worker_threads"); > new Worker("console.log(require(\"inspector\").close)", {eval:true}) undefined ``` (As far as I can tell this is intentional and has existed for quite some time.) PR-URL: https://github.com/nodejs/node/pull/43867 Reviewed-By: Luigi Pinca Reviewed-By: Feng Yu --- doc/api/inspector.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/api/inspector.md b/doc/api/inspector.md index dd5fac313d020b..10458d9ac51c82 100644 --- a/doc/api/inspector.md +++ b/doc/api/inspector.md @@ -19,6 +19,8 @@ const inspector = require('node:inspector'); Deactivate the inspector. Blocks until there are no active connections. +This function is not available in [worker threads][]. + ## `inspector.console` * {Object} An object to send messages to the remote inspector console. @@ -254,3 +256,4 @@ session.post('HeapProfiler.takeHeapSnapshot', null, (err, r) => { [`'Debugger.paused'`]: https://chromedevtools.github.io/devtools-protocol/v8/Debugger#event-paused [`session.connect()`]: #sessionconnect [security warning]: cli.md#warning-binding-inspector-to-a-public-ipport-combination-is-insecure +[worker threads]: worker_threads.md From 23cab44da22a113d322603df289dd6c00c157486 Mon Sep 17 00:00:00 2001 From: Kohei Ueno Date: Tue, 26 Jul 2022 00:12:05 +0900 Subject: [PATCH 44/46] test: update WPT encoding tests PR-URL: https://github.com/nodejs/node/pull/43958 Reviewed-By: Luigi Pinca Reviewed-By: Feng Yu Reviewed-By: Darshan Sen --- test/fixtures/wpt/README.md | 2 +- test/fixtures/wpt/encoding/encodeInto.any.js | 25 ++-- .../wpt/encoding/encodeInto.any.js.headers | 2 + .../encoding/idlharness-shadowrealm.window.js | 2 + .../wpt/encoding/sharedarraybuffer.https.html | 14 +++ .../sharedarraybuffer.https.html.headers | 2 + .../encoding/single-byte-decoder.window.js | 107 ++++++++++++++++++ .../wpt/encoding/streams/decode-utf8.any.js | 11 +- .../streams/decode-utf8.any.js.headers | 2 + .../encoding/textdecoder-copy.any.js.headers | 2 + .../wpt/encoding/textdecoder-eof.any.js | 40 +++++++ .../wpt/encoding/textdecoder-streaming.any.js | 53 ++++++++- .../textdecoder-streaming.any.js.headers | 2 + test/fixtures/wpt/versions.json | 2 +- test/wpt/status/encoding.json | 15 ++- 15 files changed, 259 insertions(+), 22 deletions(-) create mode 100644 test/fixtures/wpt/encoding/encodeInto.any.js.headers create mode 100644 test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js create mode 100644 test/fixtures/wpt/encoding/sharedarraybuffer.https.html create mode 100644 test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers create mode 100644 test/fixtures/wpt/encoding/single-byte-decoder.window.js create mode 100644 test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers create mode 100644 test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers create mode 100644 test/fixtures/wpt/encoding/textdecoder-eof.any.js create mode 100644 test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers diff --git a/test/fixtures/wpt/README.md b/test/fixtures/wpt/README.md index 3896de60f84a73..525476728748b0 100644 --- a/test/fixtures/wpt/README.md +++ b/test/fixtures/wpt/README.md @@ -14,7 +14,7 @@ Last update: - console: https://github.com/web-platform-tests/wpt/tree/3b1f72e99a/console - dom/abort: https://github.com/web-platform-tests/wpt/tree/c49cafb491/dom/abort - dom/events: https://github.com/web-platform-tests/wpt/tree/f8821adb28/dom/events -- encoding: https://github.com/web-platform-tests/wpt/tree/35f70910d3/encoding +- encoding: https://github.com/web-platform-tests/wpt/tree/c1b24fce6e/encoding - FileAPI: https://github.com/web-platform-tests/wpt/tree/3b279420d4/FileAPI - hr-time: https://github.com/web-platform-tests/wpt/tree/9910784394/hr-time - html/webappapis/atob: https://github.com/web-platform-tests/wpt/tree/f267e1dca6/html/webappapis/atob diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js b/test/fixtures/wpt/encoding/encodeInto.any.js index eca0e1bca158b0..69d7089006ec38 100644 --- a/test/fixtures/wpt/encoding/encodeInto.any.js +++ b/test/fixtures/wpt/encoding/encodeInto.any.js @@ -120,19 +120,22 @@ }); }); -[DataView, - Int8Array, - Int16Array, - Int32Array, - Uint16Array, - Uint32Array, - Uint8ClampedArray, - Float32Array, - Float64Array].forEach(view => { +["DataView", + "Int8Array", + "Int16Array", + "Int32Array", + "Uint16Array", + "Uint32Array", + "Uint8ClampedArray", + "BigInt64Array", + "BigUint64Array", + "Float32Array", + "Float64Array"].forEach(type => { ["ArrayBuffer", "SharedArrayBuffer"].forEach((arrayBufferOrSharedArrayBuffer) => { test(() => { - assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", new view(createBuffer(arrayBufferOrSharedArrayBuffer, 0)))); - }, "Invalid encodeInto() destination: " + view.name + ", backed by: " + arrayBufferOrSharedArrayBuffer); + const viewInstance = new self[type](createBuffer(arrayBufferOrSharedArrayBuffer, 0)); + assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", viewInstance)); + }, "Invalid encodeInto() destination: " + type + ", backed by: " + arrayBufferOrSharedArrayBuffer); }); }); diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js.headers b/test/fixtures/wpt/encoding/encodeInto.any.js.headers new file mode 100644 index 00000000000000..4fff9d9fba4c81 --- /dev/null +++ b/test/fixtures/wpt/encoding/encodeInto.any.js.headers @@ -0,0 +1,2 @@ +Cross-Origin-Opener-Policy: same-origin +Cross-Origin-Embedder-Policy: require-corp \ No newline at end of file diff --git a/test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js b/test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js new file mode 100644 index 00000000000000..baf3efcbb027e4 --- /dev/null +++ b/test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js @@ -0,0 +1,2 @@ +// META: script=/resources/idlharness-shadowrealm.js +idl_test_shadowrealm(["encoding"], ["streams"]); diff --git a/test/fixtures/wpt/encoding/sharedarraybuffer.https.html b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html new file mode 100644 index 00000000000000..2496edacb89ee8 --- /dev/null +++ b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html @@ -0,0 +1,14 @@ + + + + diff --git a/test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers new file mode 100644 index 00000000000000..4b06ac7cc63e8a --- /dev/null +++ b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers @@ -0,0 +1,2 @@ +Cross-Origin-Opener-Policy:same-origin +Cross-Origin-Embedder-Policy:require-corp diff --git a/test/fixtures/wpt/encoding/single-byte-decoder.window.js b/test/fixtures/wpt/encoding/single-byte-decoder.window.js new file mode 100644 index 00000000000000..6bca8e623434e9 --- /dev/null +++ b/test/fixtures/wpt/encoding/single-byte-decoder.window.js @@ -0,0 +1,107 @@ +// META: timeout=long +// META: variant=?XMLHttpRequest +// META: variant=?TextDecoder +// META: variant=?document +// META: script=resources/encodings.js + +var singleByteEncodings = encodings_table.filter(function(group) { + return group.heading === "Legacy single-byte encodings"; +})[0].encodings, +// https://encoding.spec.whatwg.org/indexes.json + singleByteIndexes = { + "IBM866":[1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,9617,9618,9619,9474,9508,9569,9570,9558,9557,9571,9553,9559,9565,9564,9563,9488,9492,9524,9516,9500,9472,9532,9566,9567,9562,9556,9577,9574,9568,9552,9580,9575,9576,9572,9573,9561,9560,9554,9555,9579,9578,9496,9484,9608,9604,9612,9616,9600,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1025,1105,1028,1108,1031,1111,1038,1118,176,8729,183,8730,8470,164,9632,160], + "ISO-8859-2":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,260,728,321,164,317,346,167,168,352,350,356,377,173,381,379,176,261,731,322,180,318,347,711,184,353,351,357,378,733,382,380,340,193,194,258,196,313,262,199,268,201,280,203,282,205,206,270,272,323,327,211,212,336,214,215,344,366,218,368,220,221,354,223,341,225,226,259,228,314,263,231,269,233,281,235,283,237,238,271,273,324,328,243,244,337,246,247,345,367,250,369,252,253,355,729], + "ISO-8859-3":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,294,728,163,164,null,292,167,168,304,350,286,308,173,null,379,176,295,178,179,180,181,293,183,184,305,351,287,309,189,null,380,192,193,194,null,196,266,264,199,200,201,202,203,204,205,206,207,null,209,210,211,212,288,214,215,284,217,218,219,220,364,348,223,224,225,226,null,228,267,265,231,232,233,234,235,236,237,238,239,null,241,242,243,244,289,246,247,285,249,250,251,252,365,349,729], + "ISO-8859-4":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,260,312,342,164,296,315,167,168,352,274,290,358,173,381,175,176,261,731,343,180,297,316,711,184,353,275,291,359,330,382,331,256,193,194,195,196,197,198,302,268,201,280,203,278,205,206,298,272,325,332,310,212,213,214,215,216,370,218,219,220,360,362,223,257,225,226,227,228,229,230,303,269,233,281,235,279,237,238,299,273,326,333,311,244,245,246,247,248,371,250,251,252,361,363,729], + "ISO-8859-5":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,173,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,8470,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,167,1118,1119], + "ISO-8859-6":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,null,null,null,164,null,null,null,null,null,null,null,1548,173,null,null,null,null,null,null,null,null,null,null,null,null,null,1563,null,null,null,1567,null,1569,1570,1571,1572,1573,1574,1575,1576,1577,1578,1579,1580,1581,1582,1583,1584,1585,1586,1587,1588,1589,1590,1591,1592,1593,1594,null,null,null,null,null,1600,1601,1602,1603,1604,1605,1606,1607,1608,1609,1610,1611,1612,1613,1614,1615,1616,1617,1618,null,null,null,null,null,null,null,null,null,null,null,null,null], + "ISO-8859-7":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,8216,8217,163,8364,8367,166,167,168,169,890,171,172,173,null,8213,176,177,178,179,900,901,902,183,904,905,906,187,908,189,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,null,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,null], + "ISO-8859-8":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,null,162,163,164,165,166,167,168,169,215,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,247,187,188,189,190,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,8215,1488,1489,1490,1491,1492,1493,1494,1495,1496,1497,1498,1499,1500,1501,1502,1503,1504,1505,1506,1507,1508,1509,1510,1511,1512,1513,1514,null,null,8206,8207,null], + "ISO-8859-10":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,260,274,290,298,296,310,167,315,272,352,358,381,173,362,330,176,261,275,291,299,297,311,183,316,273,353,359,382,8213,363,331,256,193,194,195,196,197,198,302,268,201,280,203,278,205,206,207,208,325,332,211,212,213,214,360,216,370,218,219,220,221,222,223,257,225,226,227,228,229,230,303,269,233,281,235,279,237,238,239,240,326,333,243,244,245,246,361,248,371,250,251,252,253,254,312], + "ISO-8859-13":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,8221,162,163,164,8222,166,167,216,169,342,171,172,173,174,198,176,177,178,179,8220,181,182,183,248,185,343,187,188,189,190,230,260,302,256,262,196,197,280,274,268,201,377,278,290,310,298,315,352,323,325,211,332,213,214,215,370,321,346,362,220,379,381,223,261,303,257,263,228,229,281,275,269,233,378,279,291,311,299,316,353,324,326,243,333,245,246,247,371,322,347,363,252,380,382,8217], + "ISO-8859-14":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,7682,7683,163,266,267,7690,167,7808,169,7810,7691,7922,173,174,376,7710,7711,288,289,7744,7745,182,7766,7809,7767,7811,7776,7923,7812,7813,7777,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,372,209,210,211,212,213,214,7786,216,217,218,219,220,221,374,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,373,241,242,243,244,245,246,7787,248,249,250,251,252,253,375,255], + "ISO-8859-15":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,8364,165,352,167,353,169,170,171,172,173,174,175,176,177,178,179,381,181,182,183,382,185,186,187,338,339,376,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255], + "ISO-8859-16":[128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,260,261,321,8364,8222,352,167,353,169,536,171,377,173,378,379,176,177,268,322,381,8221,182,183,382,269,537,187,338,339,376,380,192,193,194,258,196,262,198,199,200,201,202,203,204,205,206,207,272,323,210,211,212,336,214,346,368,217,218,219,220,280,538,223,224,225,226,259,228,263,230,231,232,233,234,235,236,237,238,239,273,324,242,243,244,337,246,347,369,249,250,251,252,281,539,255], + "KOI8-R":[9472,9474,9484,9488,9492,9496,9500,9508,9516,9524,9532,9600,9604,9608,9612,9616,9617,9618,9619,8992,9632,8729,8730,8776,8804,8805,160,8993,176,178,183,247,9552,9553,9554,1105,9555,9556,9557,9558,9559,9560,9561,9562,9563,9564,9565,9566,9567,9568,9569,1025,9570,9571,9572,9573,9574,9575,9576,9577,9578,9579,9580,169,1102,1072,1073,1094,1076,1077,1092,1075,1093,1080,1081,1082,1083,1084,1085,1086,1087,1103,1088,1089,1090,1091,1078,1074,1100,1099,1079,1096,1101,1097,1095,1098,1070,1040,1041,1062,1044,1045,1060,1043,1061,1048,1049,1050,1051,1052,1053,1054,1055,1071,1056,1057,1058,1059,1046,1042,1068,1067,1047,1064,1069,1065,1063,1066], + "KOI8-U":[9472,9474,9484,9488,9492,9496,9500,9508,9516,9524,9532,9600,9604,9608,9612,9616,9617,9618,9619,8992,9632,8729,8730,8776,8804,8805,160,8993,176,178,183,247,9552,9553,9554,1105,1108,9556,1110,1111,9559,9560,9561,9562,9563,1169,1118,9566,9567,9568,9569,1025,1028,9571,1030,1031,9574,9575,9576,9577,9578,1168,1038,169,1102,1072,1073,1094,1076,1077,1092,1075,1093,1080,1081,1082,1083,1084,1085,1086,1087,1103,1088,1089,1090,1091,1078,1074,1100,1099,1079,1096,1101,1097,1095,1098,1070,1040,1041,1062,1044,1045,1060,1043,1061,1048,1049,1050,1051,1052,1053,1054,1055,1071,1056,1057,1058,1059,1046,1042,1068,1067,1047,1064,1069,1065,1063,1066], + "macintosh":[196,197,199,201,209,214,220,225,224,226,228,227,229,231,233,232,234,235,237,236,238,239,241,243,242,244,246,245,250,249,251,252,8224,176,162,163,167,8226,182,223,174,169,8482,180,168,8800,198,216,8734,177,8804,8805,165,181,8706,8721,8719,960,8747,170,186,937,230,248,191,161,172,8730,402,8776,8710,171,187,8230,160,192,195,213,338,339,8211,8212,8220,8221,8216,8217,247,9674,255,376,8260,8364,8249,8250,64257,64258,8225,183,8218,8222,8240,194,202,193,203,200,205,206,207,204,211,212,63743,210,218,219,217,305,710,732,175,728,729,730,184,733,731,711], + "windows-874":[8364,129,130,131,132,8230,134,135,136,137,138,139,140,141,142,143,144,8216,8217,8220,8221,8226,8211,8212,152,153,154,155,156,157,158,159,160,3585,3586,3587,3588,3589,3590,3591,3592,3593,3594,3595,3596,3597,3598,3599,3600,3601,3602,3603,3604,3605,3606,3607,3608,3609,3610,3611,3612,3613,3614,3615,3616,3617,3618,3619,3620,3621,3622,3623,3624,3625,3626,3627,3628,3629,3630,3631,3632,3633,3634,3635,3636,3637,3638,3639,3640,3641,3642,null,null,null,null,3647,3648,3649,3650,3651,3652,3653,3654,3655,3656,3657,3658,3659,3660,3661,3662,3663,3664,3665,3666,3667,3668,3669,3670,3671,3672,3673,3674,3675,null,null,null,null], + "windows-1250":[8364,129,8218,131,8222,8230,8224,8225,136,8240,352,8249,346,356,381,377,144,8216,8217,8220,8221,8226,8211,8212,152,8482,353,8250,347,357,382,378,160,711,728,321,164,260,166,167,168,169,350,171,172,173,174,379,176,177,731,322,180,181,182,183,184,261,351,187,317,733,318,380,340,193,194,258,196,313,262,199,268,201,280,203,282,205,206,270,272,323,327,211,212,336,214,215,344,366,218,368,220,221,354,223,341,225,226,259,228,314,263,231,269,233,281,235,283,237,238,271,273,324,328,243,244,337,246,247,345,367,250,369,252,253,355,729], + "windows-1251":[1026,1027,8218,1107,8222,8230,8224,8225,8364,8240,1033,8249,1034,1036,1035,1039,1106,8216,8217,8220,8221,8226,8211,8212,152,8482,1113,8250,1114,1116,1115,1119,160,1038,1118,1032,164,1168,166,167,1025,169,1028,171,172,173,174,1031,176,177,1030,1110,1169,181,182,183,1105,8470,1108,187,1112,1029,1109,1111,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103], + "windows-1252":[8364,129,8218,402,8222,8230,8224,8225,710,8240,352,8249,338,141,381,143,144,8216,8217,8220,8221,8226,8211,8212,732,8482,353,8250,339,157,382,376,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255], + "windows-1253":[8364,129,8218,402,8222,8230,8224,8225,136,8240,138,8249,140,141,142,143,144,8216,8217,8220,8221,8226,8211,8212,152,8482,154,8250,156,157,158,159,160,901,902,163,164,165,166,167,168,169,null,171,172,173,174,8213,176,177,178,179,900,181,182,183,904,905,906,187,908,189,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,null,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,null], + "windows-1254":[8364,129,8218,402,8222,8230,8224,8225,710,8240,352,8249,338,141,142,143,144,8216,8217,8220,8221,8226,8211,8212,732,8482,353,8250,339,157,158,376,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,286,209,210,211,212,213,214,215,216,217,218,219,220,304,350,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,287,241,242,243,244,245,246,247,248,249,250,251,252,305,351,255], + "windows-1255":[8364,129,8218,402,8222,8230,8224,8225,710,8240,138,8249,140,141,142,143,144,8216,8217,8220,8221,8226,8211,8212,732,8482,154,8250,156,157,158,159,160,161,162,163,8362,165,166,167,168,169,215,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,247,187,188,189,190,191,1456,1457,1458,1459,1460,1461,1462,1463,1464,1465,1466,1467,1468,1469,1470,1471,1472,1473,1474,1475,1520,1521,1522,1523,1524,null,null,null,null,null,null,null,1488,1489,1490,1491,1492,1493,1494,1495,1496,1497,1498,1499,1500,1501,1502,1503,1504,1505,1506,1507,1508,1509,1510,1511,1512,1513,1514,null,null,8206,8207,null], + "windows-1256":[8364,1662,8218,402,8222,8230,8224,8225,710,8240,1657,8249,338,1670,1688,1672,1711,8216,8217,8220,8221,8226,8211,8212,1705,8482,1681,8250,339,8204,8205,1722,160,1548,162,163,164,165,166,167,168,169,1726,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,1563,187,188,189,190,1567,1729,1569,1570,1571,1572,1573,1574,1575,1576,1577,1578,1579,1580,1581,1582,1583,1584,1585,1586,1587,1588,1589,1590,215,1591,1592,1593,1594,1600,1601,1602,1603,224,1604,226,1605,1606,1607,1608,231,232,233,234,235,1609,1610,238,239,1611,1612,1613,1614,244,1615,1616,247,1617,249,1618,251,252,8206,8207,1746], + "windows-1257":[8364,129,8218,131,8222,8230,8224,8225,136,8240,138,8249,140,168,711,184,144,8216,8217,8220,8221,8226,8211,8212,152,8482,154,8250,156,175,731,159,160,null,162,163,164,null,166,167,216,169,342,171,172,173,174,198,176,177,178,179,180,181,182,183,248,185,343,187,188,189,190,230,260,302,256,262,196,197,280,274,268,201,377,278,290,310,298,315,352,323,325,211,332,213,214,215,370,321,346,362,220,379,381,223,261,303,257,263,228,229,281,275,269,233,378,279,291,311,299,316,353,324,326,243,333,245,246,247,371,322,347,363,252,380,382,729], + "windows-1258":[8364,129,8218,402,8222,8230,8224,8225,710,8240,138,8249,338,141,142,143,144,8216,8217,8220,8221,8226,8211,8212,732,8482,154,8250,339,157,158,376,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,258,196,197,198,199,200,201,202,203,768,205,206,207,272,209,777,211,212,416,214,215,216,217,218,219,220,431,771,223,224,225,226,259,228,229,230,231,232,233,234,235,769,237,238,239,273,241,803,243,244,417,246,247,248,249,250,251,252,432,8363,255], + "x-mac-cyrillic":[1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,8224,176,1168,163,167,8226,182,1030,174,169,8482,1026,1106,8800,1027,1107,8734,177,8804,8805,1110,181,1169,1032,1028,1108,1031,1111,1033,1113,1034,1114,1112,1029,172,8730,402,8776,8710,171,187,8230,160,1035,1115,1036,1116,1109,8211,8212,8220,8221,8216,8217,247,8222,1038,1118,1039,1119,8470,1025,1105,1103,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,8364] +} + +// For TextDecoder tests +var buffer = new ArrayBuffer(255), + view = new Uint8Array(buffer) +for(var i = 0, l = view.byteLength; i < l; i++) { + view[i] = i +} + +// For XMLHttpRequest and TextDecoder tests +function assert_decode(data, encoding) { + if(encoding == "ISO-8859-8-I") { + encoding = "ISO-8859-8" + } + for(var i = 0, l = data.length; i < l; i++) { + var cp = data.charCodeAt(i), + expectedCp = (i < 0x80) ? i : singleByteIndexes[encoding][i-0x80] + if(expectedCp == null) { + expectedCp = 0xFFFD + } + assert_equals(cp, expectedCp, encoding + ":" + i) + } +} + +var subsetTest = ""; +if (location.search) { + subsetTest = location.search.substr(1); +} + +// Setting up all the tests +for(var i = 0, l = singleByteEncodings.length; i < l; i++) { + var encoding = singleByteEncodings[i] + for(var ii = 0, ll = encoding.labels.length; ii < ll; ii++) { + var label = encoding.labels[ii] + + if (subsetTest == "XMLHttpRequest" || !subsetTest) { + async_test(function(t) { + var xhr = new XMLHttpRequest, + name = encoding.name // need scoped variable + xhr.open("GET", "resources/single-byte-raw.py?label=" + label) + xhr.send(null) + xhr.onload = t.step_func_done(function() { assert_decode(xhr.responseText, name) }) + }, encoding.name + ": " + label + " (XMLHttpRequest)") + } + + if (subsetTest == "TextDecoder" || !subsetTest) { + test(function() { + var d = new TextDecoder(label), + data = d.decode(view) + assert_equals(d.encoding, encoding.name.toLowerCase()) // ASCII names only, so safe + assert_decode(data, encoding.name) + }, encoding.name + ": " + label + " (TextDecoder)") + } + + if (subsetTest == "document" || !subsetTest) { + async_test(function(t) { + var frame = document.createElement("iframe"), + name = encoding.name; + frame.src = "resources/text-plain-charset.py?label=" + label + frame.onload = t.step_func_done(function() { + assert_equals(frame.contentDocument.characterSet, name) + assert_equals(frame.contentDocument.inputEncoding, name) + }) + t.add_cleanup(function() { document.body.removeChild(frame) }) + document.body.appendChild(frame) + }, encoding.name + ": " + label + " (document.characterSet and document.inputEncoding)") + } + } +} diff --git a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js index 5abd8dcb8f9b8f..f6fceb299bdd36 100644 --- a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js +++ b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js @@ -43,7 +43,16 @@ const array = await readableStreamToArray(output); assert_array_equals(array, [expectedOutputString], 'the output should be in one chunk'); - }, 'a trailing empty chunk should be ignored- ' + arrayBufferOrSharedArrayBuffer); + }, 'a trailing empty chunk should be ignored - ' + arrayBufferOrSharedArrayBuffer); + + promise_test(async () => { + const chunk = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, 3)); + chunk.set([0xF0, 0x9F, 0x92]); + const input = readableStreamFromArray([chunk]); + const output = input.pipeThrough(new TextDecoderStream()); + const array = await readableStreamToArray(output); + assert_array_equals(array, ['\uFFFD']); + }, 'UTF-8 EOF handling - ' + arrayBufferOrSharedArrayBuffer); }); promise_test(async () => { diff --git a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers new file mode 100644 index 00000000000000..4fff9d9fba4c81 --- /dev/null +++ b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers @@ -0,0 +1,2 @@ +Cross-Origin-Opener-Policy: same-origin +Cross-Origin-Embedder-Policy: require-corp \ No newline at end of file diff --git a/test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers b/test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers new file mode 100644 index 00000000000000..4fff9d9fba4c81 --- /dev/null +++ b/test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers @@ -0,0 +1,2 @@ +Cross-Origin-Opener-Policy: same-origin +Cross-Origin-Embedder-Policy: require-corp \ No newline at end of file diff --git a/test/fixtures/wpt/encoding/textdecoder-eof.any.js b/test/fixtures/wpt/encoding/textdecoder-eof.any.js new file mode 100644 index 00000000000000..e41e326aace244 --- /dev/null +++ b/test/fixtures/wpt/encoding/textdecoder-eof.any.js @@ -0,0 +1,40 @@ +test(() => { + // Truncated sequences + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0])), "\uFFFD"); + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F])), "\uFFFD"); + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x92])), "\uFFFD"); + + // Errors near end-of-queue + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x41])), "\uFFFDA"); + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0x42])), "\uFFFDAB"); + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0xF0])), "\uFFFDA\uFFFD"); + assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x8F, 0x92])), "\uFFFD\uFFFD\uFFFD"); +}, "TextDecoder end-of-queue handling"); + +test(() => { + const decoder = new TextDecoder(); + decoder.decode(new Uint8Array([0xF0]), { stream: true }); + assert_equals(decoder.decode(), "\uFFFD"); + + decoder.decode(new Uint8Array([0xF0]), { stream: true }); + decoder.decode(new Uint8Array([0x9F]), { stream: true }); + assert_equals(decoder.decode(), "\uFFFD"); + + decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }); + assert_equals(decoder.decode(new Uint8Array([0x92])), "\uFFFD"); + + assert_equals(decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }), ""); + assert_equals(decoder.decode(new Uint8Array([0x41]), { stream: true }), "\uFFFDA"); + assert_equals(decoder.decode(), ""); + + assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0x42]), { stream: true }), "\uFFFDAB"); + assert_equals(decoder.decode(), ""); + + assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0xF0]), { stream: true }), "\uFFFDA"); + assert_equals(decoder.decode(), "\uFFFD"); + + assert_equals(decoder.decode(new Uint8Array([0xF0]), { stream: true }), ""); + assert_equals(decoder.decode(new Uint8Array([0x8F]), { stream: true }), "\uFFFD\uFFFD"); + assert_equals(decoder.decode(new Uint8Array([0x92]), { stream: true }), "\uFFFD"); + assert_equals(decoder.decode(), ""); +}, "TextDecoder end-of-queue handling using stream: true"); diff --git a/test/fixtures/wpt/encoding/textdecoder-streaming.any.js b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js index 5717c2dbe00a2a..e473a7056e24d3 100644 --- a/test/fixtures/wpt/encoding/textdecoder-streaming.any.js +++ b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js @@ -28,10 +28,11 @@ var octets = { var decoder = new TextDecoder(encoding); for (var i = 0; i < encoded.length; i += len) { var sub = []; - for (var j = i; j < encoded.length && j < i + len; ++j) + for (var j = i; j < encoded.length && j < i + len; ++j) { sub.push(encoded[j]); - var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length)); - uintArray.set(sub); + } + var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length)); + uintArray.set(sub); out += decoder.decode(uintArray, {stream: true}); } out += decoder.decode(); @@ -39,4 +40,50 @@ var octets = { }, 'Streaming decode: ' + encoding + ', ' + len + ' byte window (' + arrayBufferOrSharedArrayBuffer + ')'); } }); + + test(() => { + function bytes(byteArray) { + const view = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, byteArray.length)); + view.set(byteArray); + return view; + } + + const decoder = new TextDecoder(); + + assert_equals(decoder.decode(bytes([0xC1]), {stream: true}), "\uFFFD"); + assert_equals(decoder.decode(), ""); + + assert_equals(decoder.decode(bytes([0xF5]), {stream: true}), "\uFFFD"); + assert_equals(decoder.decode(), ""); + + assert_equals(decoder.decode(bytes([0xE0, 0x41]), {stream: true}), "\uFFFDA"); + assert_equals(decoder.decode(bytes([0x42])), "B"); + + assert_equals(decoder.decode(bytes([0xE0, 0x80]), {stream: true}), "\uFFFD\uFFFD"); + assert_equals(decoder.decode(bytes([0x80])), "\uFFFD"); + + assert_equals(decoder.decode(bytes([0xED, 0xA0]), {stream: true}), "\uFFFD\uFFFD"); + assert_equals(decoder.decode(bytes([0x80])), "\uFFFD"); + + assert_equals(decoder.decode(bytes([0xF0, 0x41]), {stream: true}), "\uFFFDA"); + assert_equals(decoder.decode(bytes([0x42]), {stream: true}), "B"); + assert_equals(decoder.decode(bytes([0x43])), "C"); + + assert_equals(decoder.decode(bytes([0xF0, 0x80]), {stream: true}), "\uFFFD\uFFFD"); + assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD"); + assert_equals(decoder.decode(bytes([0x80])), "\uFFFD"); + + assert_equals(decoder.decode(bytes([0xF4, 0xA0]), {stream: true}), "\uFFFD\uFFFD"); + assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD"); + assert_equals(decoder.decode(bytes([0x80])), "\uFFFD"); + + assert_equals(decoder.decode(bytes([0xF0, 0x90, 0x41]), {stream: true}), "\uFFFDA"); + assert_equals(decoder.decode(bytes([0x42])), "B"); + + // 4-byte UTF-8 sequences always correspond to non-BMP characters. Here + // we make sure that, although the first 3 bytes are enough to emit the + // lead surrogate, it only gets emitted when the fourth byte is read. + assert_equals(decoder.decode(bytes([0xF0, 0x9F, 0x92]), {stream: true}), ""); + assert_equals(decoder.decode(bytes([0xA9])), "\u{1F4A9}"); + }, `Streaming decode: UTF-8 chunk tests (${arrayBufferOrSharedArrayBuffer})`); }) diff --git a/test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers new file mode 100644 index 00000000000000..4fff9d9fba4c81 --- /dev/null +++ b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers @@ -0,0 +1,2 @@ +Cross-Origin-Opener-Policy: same-origin +Cross-Origin-Embedder-Policy: require-corp \ No newline at end of file diff --git a/test/fixtures/wpt/versions.json b/test/fixtures/wpt/versions.json index 9bf84010b50958..210c91988e28ad 100644 --- a/test/fixtures/wpt/versions.json +++ b/test/fixtures/wpt/versions.json @@ -16,7 +16,7 @@ "path": "dom/events" }, "encoding": { - "commit": "35f70910d3753c8b650fdfd4c716caedfefe88c9", + "commit": "c1b24fce6e625c1b79124a58f27bf9adce02d5d7", "path": "encoding" }, "FileAPI": { diff --git a/test/wpt/status/encoding.json b/test/wpt/status/encoding.json index 88373a1ee38fc5..15dad0b2d4f8a0 100644 --- a/test/wpt/status/encoding.json +++ b/test/wpt/status/encoding.json @@ -15,8 +15,7 @@ "requires": ["small-icu"] }, "textdecoder-streaming.any.js": { - "requires": ["small-icu"], - "fail": "WebAssembly.Memory does not support shared:true" + "requires": ["small-icu"] }, "textdecoder-utf16-surrogates.any.js": { "requires": ["small-icu"] @@ -53,10 +52,10 @@ "fail": "No implementation of TextDecoderStream and TextEncoderStream" }, "encodeInto.any.js": { - "fail": "WebAssembly.Memory does not support shared:true" + "requires": ["small-icu"] }, "textdecoder-copy.any.js": { - "fail": "WebAssembly.Memory does not support shared:true" + "requires": ["small-icu"] }, "legacy-mb-schinese/gbk/gbk-decoder.any.js": { "requires": ["full-icu"], @@ -67,6 +66,12 @@ "skip": "The gb18030 encoding is not supported" }, "textdecoder-arguments.any.js": { - "fail": "Does not support flushing an incomplete sequence" + "requires": ["small-icu"] + }, + "single-byte-decoder.window.js": { + "skip": "location is not defined" + }, + "textdecoder-eof.any.js": { + "requires": ["small-icu"] } } From acc68c92ce84690f1e2c97787091050804bf08ee Mon Sep 17 00:00:00 2001 From: Bethany Nicolle Griggs Date: Mon, 25 Jul 2022 20:26:12 +0100 Subject: [PATCH 45/46] doc: update email and mailmap for BethGriggs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/43985 Reviewed-By: Tobias Nießen Reviewed-By: Michaël Zasso Reviewed-By: Richard Lau Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca --- .mailmap | 3 ++- README.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.mailmap b/.mailmap index 91a68718411943..f922b76746784f 100644 --- a/.mailmap +++ b/.mailmap @@ -67,7 +67,8 @@ Benjamin Waters Bert Belder Bert Belder Bert Belder -Beth Griggs +Beth Griggs +Beth Griggs Bidisha Pyne bl-ue <54780737+bl-ue@users.noreply.github.com> Brad Decker diff --git a/README.md b/README.md index 154a1ddebbd99a..5b6347020b9696 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ For information about the governance of the Node.js project, see * [apapirovski](https://github.com/apapirovski) - **Anatoli Papirovski** <> (he/him) * [BethGriggs](https://github.com/BethGriggs) - - **Beth Griggs** <> (she/her) + **Beth Griggs** <> (she/her) * [BridgeAR](https://github.com/BridgeAR) - **Ruben Bridgewater** <> (he/him) * [ChALkeR](https://github.com/ChALkeR) - @@ -279,7 +279,7 @@ For information about the governance of the Node.js project, see * [benjamingr](https://github.com/benjamingr) - **Benjamin Gruenbaum** <> * [BethGriggs](https://github.com/BethGriggs) - - **Beth Griggs** <> (she/her) + **Beth Griggs** <> (she/her) * [bmeck](https://github.com/bmeck) - **Bradley Farias** <> * [bnb](https://github.com/bnb) - @@ -677,7 +677,7 @@ maintaining the Node.js project. Primary GPG keys for Node.js Releasers (some Releasers sign with subkeys): -* **Beth Griggs** <> +* **Beth Griggs** <> `4ED778F539E3634C779C87C6D7062848A1AB005C` * **Bryan English** <> `141F07595B7B3FFE74309A937405533BE57C7D57` From 1073beeba45c6c567b93f751a4c20879b6185fd4 Mon Sep 17 00:00:00 2001 From: Darshan Sen Date: Fri, 15 Jul 2022 09:40:56 +0530 Subject: [PATCH 46/46] src: pass only Isolate* and env_vars to EnabledDebugList::Parse() The function doesn't require access to the entire Environment object, so this change just passes what it needs. Addresses this TODO - https://github.com/nodejs/node/blob/71691e53d601a4cf3eab3b33cebe4cc9f50c8470/src/env.cc#L372-L374 Signed-off-by: Darshan Sen PR-URL: https://github.com/nodejs/node/pull/43668 Reviewed-By: Chengzhong Wu Reviewed-By: Joyee Cheung --- src/debug_utils.cc | 6 ++++-- src/debug_utils.h | 10 ++++++---- src/env.cc | 5 +---- src/env.h | 28 ---------------------------- src/node.cc | 2 +- src/node_credentials.cc | 21 ++++++++++++--------- src/node_internals.h | 5 ++++- src/util.h | 28 ++++++++++++++++++++++++++++ 8 files changed, 56 insertions(+), 49 deletions(-) diff --git a/src/debug_utils.cc b/src/debug_utils.cc index c4c476942eee77..f721a672f10e67 100644 --- a/src/debug_utils.cc +++ b/src/debug_utils.cc @@ -1,6 +1,7 @@ #include "debug_utils-inl.h" // NOLINT(build/include) #include "env-inl.h" #include "node_internals.h" +#include "util.h" #ifdef __POSIX__ #if defined(__linux__) @@ -58,9 +59,10 @@ namespace per_process { EnabledDebugList enabled_debug_list; } -void EnabledDebugList::Parse(Environment* env) { +void EnabledDebugList::Parse(std::shared_ptr env_vars, + v8::Isolate* isolate) { std::string cats; - credentials::SafeGetenv("NODE_DEBUG_NATIVE", &cats, env); + credentials::SafeGetenv("NODE_DEBUG_NATIVE", &cats, env_vars, isolate); Parse(cats, true); } diff --git a/src/debug_utils.h b/src/debug_utils.h index bd1fa5207f9520..19847e46549263 100644 --- a/src/debug_utils.h +++ b/src/debug_utils.h @@ -4,6 +4,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS #include "async_wrap.h" +#include "util.h" #include #include @@ -66,10 +67,11 @@ class NODE_EXTERN_PRIVATE EnabledDebugList { return enabled_[static_cast(category)]; } - // Uses NODE_DEBUG_NATIVE to initialize the categories. When env is not a - // nullptr, the environment variables set in the Environment are used. - // Otherwise the system environment variables are used. - void Parse(Environment* env); + // Uses NODE_DEBUG_NATIVE to initialize the categories. The env_vars variable + // is parsed if it is not a nullptr, otherwise the system environment + // variables are parsed. + void Parse(std::shared_ptr env_vars = nullptr, + v8::Isolate* isolate = nullptr); private: // Set all categories matching cats to the value of enabled. diff --git a/src/env.cc b/src/env.cc index bc7b4f493092e0..22be69ec30a5b8 100644 --- a/src/env.cc +++ b/src/env.cc @@ -760,10 +760,7 @@ Environment::Environment(IsolateData* isolate_data, } set_env_vars(per_process::system_environment); - // TODO(joyeecheung): pass Isolate* and env_vars to it instead of the entire - // env, when the recursive dependency inclusion in "debug-utils.h" is - // resolved. - enabled_debug_list_.Parse(this); + enabled_debug_list_.Parse(env_vars(), isolate); // We create new copies of the per-Environment option sets, so that it is // easier to modify them after Environment creation. The defaults are diff --git a/src/env.h b/src/env.h index b2ff65d8b13e39..ae234a99eb2d5f 100644 --- a/src/env.h +++ b/src/env.h @@ -656,34 +656,6 @@ struct ContextInfo { class EnabledDebugList; -class KVStore { - public: - KVStore() = default; - virtual ~KVStore() = default; - KVStore(const KVStore&) = delete; - KVStore& operator=(const KVStore&) = delete; - KVStore(KVStore&&) = delete; - KVStore& operator=(KVStore&&) = delete; - - virtual v8::MaybeLocal Get(v8::Isolate* isolate, - v8::Local key) const = 0; - virtual v8::Maybe Get(const char* key) const = 0; - virtual void Set(v8::Isolate* isolate, - v8::Local key, - v8::Local value) = 0; - virtual int32_t Query(v8::Isolate* isolate, - v8::Local key) const = 0; - virtual int32_t Query(const char* key) const = 0; - virtual void Delete(v8::Isolate* isolate, v8::Local key) = 0; - virtual v8::Local Enumerate(v8::Isolate* isolate) const = 0; - - virtual std::shared_ptr Clone(v8::Isolate* isolate) const; - virtual v8::Maybe AssignFromObject(v8::Local context, - v8::Local entries); - - static std::shared_ptr CreateMapKVStore(); -}; - namespace per_process { extern std::shared_ptr system_environment; } diff --git a/src/node.cc b/src/node.cc index bae40383c9d6ae..5ac6ca75389d19 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1016,7 +1016,7 @@ InitializationResult InitializeOncePerProcess( // Initialized the enabled list for Debug() calls with system // environment variables. - per_process::enabled_debug_list.Parse(nullptr); + per_process::enabled_debug_list.Parse(); atexit(ResetStdio); diff --git a/src/node_credentials.cc b/src/node_credentials.cc index fa3dfa48a3ceb2..511ea3b01408eb 100644 --- a/src/node_credentials.cc +++ b/src/node_credentials.cc @@ -34,22 +34,25 @@ bool linux_at_secure = false; namespace credentials { // Look up environment variable unless running as setuid root. -bool SafeGetenv(const char* key, std::string* text, Environment* env) { +bool SafeGetenv(const char* key, + std::string* text, + std::shared_ptr env_vars, + v8::Isolate* isolate) { #if !defined(__CloudABI__) && !defined(_WIN32) if (per_process::linux_at_secure || getuid() != geteuid() || getgid() != getegid()) goto fail; #endif - if (env != nullptr) { - HandleScope handle_scope(env->isolate()); - TryCatch ignore_errors(env->isolate()); - MaybeLocal maybe_value = env->env_vars()->Get( - env->isolate(), - String::NewFromUtf8(env->isolate(), key).ToLocalChecked()); + if (env_vars != nullptr) { + DCHECK_NOT_NULL(isolate); + HandleScope handle_scope(isolate); + TryCatch ignore_errors(isolate); + MaybeLocal maybe_value = env_vars->Get( + isolate, String::NewFromUtf8(isolate, key).ToLocalChecked()); Local value; if (!maybe_value.ToLocal(&value)) goto fail; - String::Utf8Value utf8_value(env->isolate(), value); + String::Utf8Value utf8_value(isolate, value); if (*utf8_value == nullptr) goto fail; *text = std::string(*utf8_value, utf8_value.length()); return true; @@ -86,7 +89,7 @@ static void SafeGetenv(const FunctionCallbackInfo& args) { Isolate* isolate = env->isolate(); Utf8Value strenvtag(isolate, args[0]); std::string text; - if (!SafeGetenv(*strenvtag, &text, env)) return; + if (!SafeGetenv(*strenvtag, &text, env->env_vars(), isolate)) return; Local result = ToV8Value(isolate->GetCurrentContext(), text).ToLocalChecked(); args.GetReturnValue().Set(result); diff --git a/src/node_internals.h b/src/node_internals.h index ed208cca7046fa..f7314c906e5806 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -290,7 +290,10 @@ class ThreadPoolWork { #endif // defined(__POSIX__) && !defined(__ANDROID__) && !defined(__CloudABI__) namespace credentials { -bool SafeGetenv(const char* key, std::string* text, Environment* env = nullptr); +bool SafeGetenv(const char* key, + std::string* text, + std::shared_ptr env_vars = nullptr, + v8::Isolate* isolate = nullptr); } // namespace credentials void DefineZlibConstants(v8::Local target); diff --git a/src/util.h b/src/util.h index bc1f798030e96e..e0175bbe560e3b 100644 --- a/src/util.h +++ b/src/util.h @@ -271,6 +271,34 @@ template constexpr ContainerOfHelper ContainerOf(Inner Outer::*field, Inner* pointer); +class KVStore { + public: + KVStore() = default; + virtual ~KVStore() = default; + KVStore(const KVStore&) = delete; + KVStore& operator=(const KVStore&) = delete; + KVStore(KVStore&&) = delete; + KVStore& operator=(KVStore&&) = delete; + + virtual v8::MaybeLocal Get(v8::Isolate* isolate, + v8::Local key) const = 0; + virtual v8::Maybe Get(const char* key) const = 0; + virtual void Set(v8::Isolate* isolate, + v8::Local key, + v8::Local value) = 0; + virtual int32_t Query(v8::Isolate* isolate, + v8::Local key) const = 0; + virtual int32_t Query(const char* key) const = 0; + virtual void Delete(v8::Isolate* isolate, v8::Local key) = 0; + virtual v8::Local Enumerate(v8::Isolate* isolate) const = 0; + + virtual std::shared_ptr Clone(v8::Isolate* isolate) const; + virtual v8::Maybe AssignFromObject(v8::Local context, + v8::Local entries); + + static std::shared_ptr CreateMapKVStore(); +}; + // Convenience wrapper around v8::String::NewFromOneByte(). inline v8::Local OneByteString(v8::Isolate* isolate, const char* data,