From a1da012f6ba8f3b8eb533127c4911fee522fe78d Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 14 Jul 2020 17:21:03 +0200 Subject: [PATCH 001/492] src: do not crash if ToggleAsyncHook fails during termination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the termination case, we should not crash. There’s also no harm being done by ignoring the termination exception here, since the thread is about to be torn down anyway. Also, add a guard against running this during shutdown. That is the likely cause of https://github.com/nodejs/node/issues/34361. Fixes: https://github.com/nodejs/node/issues/34361 PR-URL: https://github.com/nodejs/node/pull/34362 Fixes: https://github.com/nodejs/node/issues/27261 Reviewed-By: Gus Caplan Reviewed-By: Joyee Cheung Reviewed-By: James M Snell --- src/inspector_agent.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index 7712a62d7cbeb0..54849014a90dd4 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -928,13 +928,18 @@ void Agent::DisableAsyncHook() { void Agent::ToggleAsyncHook(Isolate* isolate, const Global& fn) { + // Guard against running this during cleanup -- no async events will be + // emitted anyway at that point anymore, and calling into JS is not possible. + // This should probably not be something we're attempting in the first place, + // Refs: https://github.com/nodejs/node/pull/34362#discussion_r456006039 + if (!parent_env_->can_call_into_js()) return; CHECK(parent_env_->has_run_bootstrapping_code()); HandleScope handle_scope(isolate); CHECK(!fn.IsEmpty()); auto context = parent_env_->context(); v8::TryCatch try_catch(isolate); USE(fn.Get(isolate)->Call(context, Undefined(isolate), 0, nullptr)); - if (try_catch.HasCaught()) { + if (try_catch.HasCaught() && !try_catch.HasTerminated()) { PrintCaughtException(isolate, context, try_catch); FatalError("\nnode::inspector::Agent::ToggleAsyncHook", "Cannot toggle Inspector's AsyncHook, please report this."); From d73b8346b8039990d225242bd5c97b49394f3bc9 Mon Sep 17 00:00:00 2001 From: ConorDavenport Date: Thu, 27 Feb 2020 11:41:05 +0000 Subject: [PATCH 002/492] assert: port common.mustCall() to assert Fixes: https://github.com/nodejs/node/issues/31392 PR-URL: https://github.com/nodejs/node/pull/31982 Reviewed-By: James M Snell Reviewed-By: Matteo Collina Reviewed-By: Zeyu Yang Reviewed-By: Colin Ihrig Reviewed-By: Denys Otrishko --- doc/api/assert.md | 135 ++++++++++++++++++ doc/api/errors.md | 7 + lib/assert.js | 3 + lib/internal/assert/assertion_error.js | 20 ++- lib/internal/assert/calltracker.js | 93 ++++++++++++ lib/internal/errors.js | 2 + node.gyp | 1 + .../parallel/test-assert-calltracker-calls.js | 66 +++++++++ .../test-assert-calltracker-report.js | 32 +++++ .../test-assert-calltracker-verify.js | 32 +++++ 10 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 lib/internal/assert/calltracker.js create mode 100644 test/parallel/test-assert-calltracker-calls.js create mode 100644 test/parallel/test-assert-calltracker-report.js create mode 100644 test/parallel/test-assert-calltracker-verify.js diff --git a/doc/api/assert.md b/doc/api/assert.md index 61572f2bd05f9f..c027df2497f15c 100644 --- a/doc/api/assert.md +++ b/doc/api/assert.md @@ -147,6 +147,137 @@ try { } ``` +## Class: `assert.CallTracker` + +### `new assert.CallTracker()` + + +Creates a new [`CallTracker`][] object which can be used to track if functions +were called a specific number of times. The `tracker.verify()` must be called +for the verification to take place. The usual pattern would be to call it in a +[`process.on('exit')`][] handler. + +```js +const assert = require('assert'); + +const tracker = new assert.CallTracker(); + +function func() {} + +// callsfunc() must be called exactly 1 time before tracker.verify(). +const callsfunc = tracker.calls(func, 1); + +callsfunc(); + +// Calls tracker.verify() and verifies if all tracker.calls() functions have +// been called exact times. +process.on('exit', () => { + tracker.verify(); +}); +``` + +### `tracker.calls([fn][, exact])` + + +* `fn` {Function} **Default** A no-op function. +* `exact` {number} **Default** `1`. +* Returns: {Function} that wraps `fn`. + +The wrapper function is expected to be called exactly `exact` times. If the +function has not been called exactly `exact` times when +[`tracker.verify()`][] is called, then [`tracker.verify()`][] will throw an +error. + +```js +const assert = require('assert'); + +// Creates call tracker. +const tracker = new assert.CallTracker(); + +function func() {} + +// Returns a function that wraps func() that must be called exact times +// before tracker.verify(). +const callsfunc = tracker.calls(func); +``` + +### `tracker.report()` + + +* Returns: {Array} of objects containing information about the wrapper functions +returned by [`tracker.calls()`][]. +* Object {Object} + * `message` {string} + * `actual` {number} The actual number of times the function was called. + * `expected` {number} The number of times the function was expected to be + called. + * `operator` {string} The name of the function that is wrapped. + * `stack` {Object} A stack trace of the function. + +The arrays contains information about the expected and actual number of calls of +the functions that have not been called the expected number of times. + +```js +const assert = require('assert'); + +// Creates call tracker. +const tracker = new assert.CallTracker(); + +function func() {} + +function foo() {} + +// Returns a function that wraps func() that must be called exact times +// before tracker.verify(). +const callsfunc = tracker.calls(func, 2); + +// Returns an array containing information on callsfunc() +tracker.report(); +// [ +// { +// message: 'Expected the func function to be executed 2 time(s) but was +// executed 0 time(s).', +// actual: 0, +// expected: 2, +// operator: 'func', +// stack: stack trace +// } +// ] +``` + +### `tracker.verify()` + + +Iterates through the list of functions passed to +[`tracker.calls()`][] and will throw an error for functions that +have not been called the expected number of times. + +```js +const assert = require('assert'); + +// Creates call tracker. +const tracker = new assert.CallTracker(); + +function func() {} + +// Returns a function that wraps func() that must be called exact times +// before tracker.verify(). +const callsfunc = tracker.calls(func, 2); + +callsfunc(); + +// Will throw an error since callsfunc() was only called once. +tracker.verify(); +``` + ## `assert(value[, message])` + +Type: Documentation-only + +Calling `process.umask()` with no arguments causes the process-wide umask to be +written twice. This introduces a race condition between threads, and is a +potential security vulnerability. There is no safe, cross-platform alternative +API. + [`--http-parser=legacy`]: cli.html#cli_http_parser_library [`--pending-deprecation`]: cli.html#cli_pending_deprecation [`--throw-deprecation`]: cli.html#cli_throw_deprecation diff --git a/doc/api/process.md b/doc/api/process.md index 2fb0b77f7df702..d9b776fdbc9f04 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -2396,8 +2396,18 @@ flag's behavior. ## `process.umask([mask])` +> Stability: 0 - Deprecated. Calling `process.umask()` with no arguments is +> deprecated. No alternative is provided. + * `mask` {string|integer} The `process.umask()` method sets or returns the Node.js process's file mode From 79c4c73f4c206a6ba26a6a404f8b41151a32f22a Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Tue, 7 Apr 2020 16:17:14 -0700 Subject: [PATCH 007/492] doc: split process.umask() entry into two Split doc entries for process.umask() into one entry for process.umask() (which is deprecated) and another for `process.umask(mask)` which is not deprecated. Backport-PR-URL: https://github.com/nodejs/node/pull/34591 PR-URL: https://github.com/nodejs/node/pull/32711 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell --- doc/api/deprecations.md | 4 ++-- doc/api/process.md | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md index c850c5c16cf781..c2bac5198243f4 100644 --- a/doc/api/deprecations.md +++ b/doc/api/deprecations.md @@ -2547,15 +2547,15 @@ To maintain existing behaviour `response.finished` should be replaced with Type: Documentation-only -Calling `process.umask()` with no arguments causes the process-wide umask to be +Calling `process.umask()` with no argument causes the process-wide umask to be written twice. This introduces a race condition between threads, and is a potential security vulnerability. There is no safe, cross-platform alternative API. diff --git a/doc/api/process.md b/doc/api/process.md index d9b776fdbc9f04..33a3debc0b53ea 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -2393,27 +2393,35 @@ documentation for the [`'warning'` event][process_warning] and the [`emitWarning()` method][process_emit_warning] for more information about this flag's behavior. -## `process.umask([mask])` +## `process.umask()` -> Stability: 0 - Deprecated. Calling `process.umask()` with no arguments is -> deprecated. No alternative is provided. +> Stability: 0 - Deprecated. Calling `process.umask()` with no argument causes +> the process-wide umask to be written twice. This introduces a race condition +> between threads, and is a potential security vulnerability. There is no safe, +> cross-platform alternative API. + +`process.umask()` returns the Node.js process's file mode creation mask. Child +processes inherit the mask from the parent process. + +## `process.umask(mask)` + * `mask` {string|integer} -The `process.umask()` method sets or returns the Node.js process's file mode -creation mask. Child processes inherit the mask from the parent process. Invoked -without an argument, the current mask is returned, otherwise the umask is set to -the argument value and the previous mask is returned. +`process.umask(mask)` sets the Node.js process's file mode creation mask. Child +processes inherit the mask from the parent process. Returns the previous mask. ```js const newmask = 0o022; @@ -2423,8 +2431,7 @@ console.log( ); ``` -[`Worker`][] threads are able to read the umask, however attempting to set the -umask will result in a thrown exception. +In [`Worker`][] threads, `process.umask(mask)` will throw an exception. ## `process.uptime()` + +Type: Documentation-only + +A CommonJS module can access the first module that required it using +`module.parent`. This feature is deprecated because it does not work +consistently in the presence of ECMAScript modules and because it gives an +inaccurate representation of the CommonJS module graph. + +Some modules use it to check if they are the entry point of the current process. +Instead, it is recommended to compare `require.main` and `module`: + +```js +if (require.main === module) { + // Code section that will run only if current file is the entry point. +} +``` + +When looking for the CommonJS modules that have required the current one, +`require.cache` and `module.children` can be used: + +```js +const moduleParents = Object.values(require.cache) + .filter((m) => m.children.includes(module)); +``` + [`--http-parser=legacy`]: cli.html#cli_http_parser_library [`--pending-deprecation`]: cli.html#cli_pending_deprecation [`--throw-deprecation`]: cli.html#cli_throw_deprecation diff --git a/doc/api/modules.md b/doc/api/modules.md index 7cb5de8c4589e2..e8215a2ace67ed 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -894,11 +894,19 @@ loading. ### `module.parent` -* {module} +> Stability: 0 - Deprecated: Please use [`require.main`][] and +> [`module.children`][] instead. + +* {module | null | undefined} -The module that first required this one. +The module that first required this one, or `null` if the current module is the +entry point of the current process, or `undefined` if the module was loaded by +something that is not a CommonJS module (E.G.: REPL or `import`). ### `module.path` diff --git a/doc/api/stream.md b/doc/api/stream.md index b8c864c7913cd0..e4d79623f2724e 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -1739,7 +1739,7 @@ Custom `Writable` streams *must* call the `new stream.Writable([options])` constructor and implement the `writable._write()` and/or `writable._writev()` method. -#### Constructor: `new stream.Writable([options])` +#### `new stream.Writable([options])` From ff7fbc38f170d631de294e0dd6ee9f0180cfab0d Mon Sep 17 00:00:00 2001 From: Brian White Date: Sat, 13 Jun 2020 13:35:16 -0400 Subject: [PATCH 013/492] events: improve listeners() performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/33863 Reviewed-By: Robert Nagy Reviewed-By: Anna Henningsen Reviewed-By: Gerhard Stöbich Reviewed-By: Luigi Pinca Reviewed-By: James M Snell Reviewed-By: Yongsheng Zhang --- benchmark/events/ee-listeners.js | 2 +- lib/events.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmark/events/ee-listeners.js b/benchmark/events/ee-listeners.js index 08631e95e2165a..9de8d04a175b4b 100644 --- a/benchmark/events/ee-listeners.js +++ b/benchmark/events/ee-listeners.js @@ -14,7 +14,7 @@ function main({ n, listeners, raw }) { for (let k = 0; k < listeners; k += 1) { ee.on('dummy0', () => {}); - ee.on('dummy1', () => {}); + ee.once('dummy1', () => {}); } if (raw === 'true') { diff --git a/lib/events.js b/lib/events.js index 6717363acbff96..92f968a2b54a43 100644 --- a/lib/events.js +++ b/lib/events.js @@ -22,7 +22,6 @@ 'use strict'; const { - Array, Boolean, Error, MathMin, @@ -616,9 +615,11 @@ function arrayClone(arr) { } function unwrapListeners(arr) { - const ret = new Array(arr.length); + const ret = arrayClone(arr); for (let i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; + const orig = ret[i].listener; + if (typeof orig === 'function') + ret[i] = orig; } return ret; } From 88d12c00da8e479261e88d6fcfe81c96b7dbd369 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 16 Jun 2020 13:18:42 +0200 Subject: [PATCH 014/492] src: remove unnecessary ToLocalChecked call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/33902 Reviewed-By: Anna Henningsen Reviewed-By: Michaël Zasso Reviewed-By: Richard Lau Reviewed-By: James M Snell Reviewed-By: Tobias Nießen --- src/node_contextify.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node_contextify.cc b/src/node_contextify.cc index b90b369d1b379c..5d44c2f76dd7a0 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -1119,14 +1119,14 @@ void ContextifyContext::CompileFunction( context_extensions.size(), context_extensions.data(), options, v8::ScriptCompiler::NoCacheReason::kNoCacheNoReason, &script); - if (maybe_fn.IsEmpty()) { + Local fn; + if (!maybe_fn.ToLocal(&fn)) { if (try_catch.HasCaught() && !try_catch.HasTerminated()) { errors::DecorateErrorStack(env, try_catch); try_catch.ReThrow(); } return; } - Local fn = maybe_fn.ToLocalChecked(); Local cache_key; if (!env->compiled_fn_entry_template()->NewInstance( From 6d22ae36302a9f86d417f1eb35b1ca694b468e9d Mon Sep 17 00:00:00 2001 From: Gabriel Schulhof Date: Tue, 16 Jun 2020 12:27:14 -0700 Subject: [PATCH 015/492] doc: document n-api callback scope usage Document that it is not necessary to open handle and/or callback scopes inside finalizer, async work, thread-safe function etc. callbacks unless for reasons documented in the section about object lifetime management. Link usage of callback signatures to their definition. Fixes: https://github.com/nodejs/node/issues/33893 PR-URL: https://github.com/nodejs/node/pull/33915 Reviewed-By: James M Snell Reviewed-By: Anna Henningsen Reviewed-By: Chengzhong Wu Reviewed-By: Michael Dawson --- doc/api/n-api.md | 52 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/doc/api/n-api.md b/doc/api/n-api.md index 23262c853fcf41..207f840257ebaa 100644 --- a/doc/api/n-api.md +++ b/doc/api/n-api.md @@ -389,6 +389,7 @@ napi_status napi_set_instance_data(napi_env env, * `[in] data`: The data item to make available to bindings of this instance. * `[in] finalize_cb`: The function to call when the environment is being torn down. The function receives `data` so that it might free it. + [`napi_finalize`][] provides more details. * `[in] finalize_hint`: Optional hint to pass to the finalize callback during collection. @@ -592,6 +593,7 @@ minimum lifetimes explicitly. For more details, review the [Object lifetime management][]. ### N-API callback types + #### napi_callback_info A synchronous function that disassociates a connected `dgram.Socket` from -its remote address. Trying to call `disconnect()` on an already disconnected -socket will result in an [`ERR_SOCKET_DGRAM_NOT_CONNECTED`][] exception. +its remote address. Trying to call `disconnect()` on an unbound or already +disconnected socket will result in an [`ERR_SOCKET_DGRAM_NOT_CONNECTED`][] +exception. ### `socket.dropMembership(multicastAddress[, multicastInterface])` + The `assert` module provides a set of assertion functions for verifying invariants. diff --git a/doc/api/async_hooks.md b/doc/api/async_hooks.md index d7182f43f45288..67a475a25b359b 100644 --- a/doc/api/async_hooks.md +++ b/doc/api/async_hooks.md @@ -4,6 +4,8 @@ > Stability: 1 - Experimental + + The `async_hooks` module provides an API to track asynchronous resources. It can be accessed using: diff --git a/doc/api/buffer.md b/doc/api/buffer.md index 9f0b7a391688d9..1907c6233b480d 100644 --- a/doc/api/buffer.md +++ b/doc/api/buffer.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + In Node.js, `Buffer` objects are used to represent binary data in the form of a sequence of bytes. Many Node.js APIs, for example streams and file system operations, support `Buffer`s, as interactions with the operating system or diff --git a/doc/api/child_process.md b/doc/api/child_process.md index f09768573e3142..9b816414bc6659 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `child_process` module provides the ability to spawn child processes in a manner that is similar, but not identical, to popen(3). This capability is primarily provided by the [`child_process.spawn()`][] function: diff --git a/doc/api/cluster.md b/doc/api/cluster.md index dc18eb3e0f9f72..be612297366934 100644 --- a/doc/api/cluster.md +++ b/doc/api/cluster.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + A single instance of Node.js runs in a single thread. To take advantage of multi-core systems, the user will sometimes want to launch a cluster of Node.js processes to handle the load. diff --git a/doc/api/console.md b/doc/api/console.md index 1498c6d30e5132..b97b70ca81b553 100644 --- a/doc/api/console.md +++ b/doc/api/console.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `console` module provides a simple debugging console that is similar to the JavaScript console mechanism provided by web browsers. diff --git a/doc/api/crypto.md b/doc/api/crypto.md index 72982ecbb57651..e087c12479dc25 100644 --- a/doc/api/crypto.md +++ b/doc/api/crypto.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `crypto` module provides cryptographic functionality that includes a set of wrappers for OpenSSL's hash, HMAC, cipher, decipher, sign, and verify functions. diff --git a/doc/api/dgram.md b/doc/api/dgram.md index eef779694aa14e..3a3bdc5eae3afb 100644 --- a/doc/api/dgram.md +++ b/doc/api/dgram.md @@ -6,6 +6,8 @@ + + The `dgram` module provides an implementation of UDP datagram sockets. ```js diff --git a/doc/api/dns.md b/doc/api/dns.md index 3ab77f7c057b21..021e765ba46d77 100644 --- a/doc/api/dns.md +++ b/doc/api/dns.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `dns` module enables name resolution. For example, use it to look up IP addresses of host names. diff --git a/doc/api/domain.md b/doc/api/domain.md index b9219fbb4d714e..109e41f749f1b4 100644 --- a/doc/api/domain.md +++ b/doc/api/domain.md @@ -16,6 +16,8 @@ changes: > Stability: 0 - Deprecated + + **This module is pending deprecation**. Once a replacement API has been finalized, this module will be fully deprecated. Most end users should **not** have cause to use this module. Users who absolutely must have diff --git a/doc/api/events.md b/doc/api/events.md index 309b124b87bc20..2760d6657cccef 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -6,6 +6,8 @@ + + Much of the Node.js core API is built around an idiomatic asynchronous event-driven architecture in which certain kinds of objects (called "emitters") emit named events that cause `Function` objects ("listeners") to be called. diff --git a/doc/api/fs.md b/doc/api/fs.md index f477138bbdc01c..dec67e11e9f3b6 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -6,6 +6,8 @@ + + The `fs` module provides an API for interacting with the file system in a manner closely modeled around standard POSIX functions. diff --git a/doc/api/http.md b/doc/api/http.md index fc79f432f7fb11..f372831a95a9f1 100644 --- a/doc/api/http.md +++ b/doc/api/http.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + To use the HTTP server and client one must `require('http')`. The HTTP interfaces in Node.js are designed to support many features diff --git a/doc/api/http2.md b/doc/api/http2.md index 40a107f7be2d6c..779a5b69e44c46 100644 --- a/doc/api/http2.md +++ b/doc/api/http2.md @@ -10,6 +10,8 @@ changes: > Stability: 2 - Stable + + The `http2` module provides an implementation of the [HTTP/2][] protocol. It can be accessed using: diff --git a/doc/api/https.md b/doc/api/https.md index 3d6a86d7167feb..146ee82815fe10 100644 --- a/doc/api/https.md +++ b/doc/api/https.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + HTTPS is the HTTP protocol over TLS/SSL. In Node.js this is implemented as a separate module. diff --git a/doc/api/inspector.md b/doc/api/inspector.md index 4f9501a5f861a7..f541b9eb99fb68 100644 --- a/doc/api/inspector.md +++ b/doc/api/inspector.md @@ -4,6 +4,8 @@ > Stability: 1 - Experimental + + The `inspector` module provides an API for interacting with the V8 inspector. It can be accessed using: diff --git a/doc/api/net.md b/doc/api/net.md index 2b427b3de77139..48cf226a07fbc2 100644 --- a/doc/api/net.md +++ b/doc/api/net.md @@ -5,6 +5,8 @@ > Stability: 2 - Stable + + The `net` module provides an asynchronous network API for creating stream-based TCP or [IPC][] servers ([`net.createServer()`][]) and clients ([`net.createConnection()`][]). diff --git a/doc/api/os.md b/doc/api/os.md index 0cf85c5f96430f..276fe546f51ec8 100644 --- a/doc/api/os.md +++ b/doc/api/os.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `os` module provides operating system-related utility methods and properties. It can be accessed using: diff --git a/doc/api/path.md b/doc/api/path.md index 8349bce252a56b..14acbb1eb5f99d 100644 --- a/doc/api/path.md +++ b/doc/api/path.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `path` module provides utilities for working with file and directory paths. It can be accessed using: diff --git a/doc/api/perf_hooks.md b/doc/api/perf_hooks.md index c91d83a966f0f9..5df28047ac3f64 100644 --- a/doc/api/perf_hooks.md +++ b/doc/api/perf_hooks.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + This module provides an implementation of a subset of the W3C [Web Performance APIs][] as well as additional APIs for Node.js-specific performance measurements. diff --git a/doc/api/process.md b/doc/api/process.md index 33a3debc0b53ea..b337e228136b03 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -3,6 +3,8 @@ + + The `process` object is a `global` that provides information about, and control over, the current Node.js process. As a global, it is always available to Node.js applications without using `require()`. It can also be explicitly diff --git a/doc/api/punycode.md b/doc/api/punycode.md index 4c9dc3460a08dd..bcdfb9a167fd04 100644 --- a/doc/api/punycode.md +++ b/doc/api/punycode.md @@ -10,6 +10,8 @@ changes: > Stability: 0 - Deprecated + + **The version of the punycode module bundled in Node.js is being deprecated**. In a future major version of Node.js this module will be removed. Users currently depending on the `punycode` module should switch to using the diff --git a/doc/api/querystring.md b/doc/api/querystring.md index 0797e3ee220d53..03ee78aad0a23d 100644 --- a/doc/api/querystring.md +++ b/doc/api/querystring.md @@ -6,6 +6,8 @@ + + The `querystring` module provides utilities for parsing and formatting URL query strings. It can be accessed using: diff --git a/doc/api/readline.md b/doc/api/readline.md index c3b79d4646709d..c948e2fda5aafd 100644 --- a/doc/api/readline.md +++ b/doc/api/readline.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `readline` module provides an interface for reading data from a [Readable][] stream (such as [`process.stdin`][]) one line at a time. It can be accessed using: diff --git a/doc/api/repl.md b/doc/api/repl.md index c8420720fdc566..edd4961eea3f1a 100644 --- a/doc/api/repl.md +++ b/doc/api/repl.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `repl` module provides a Read-Eval-Print-Loop (REPL) implementation that is available both as a standalone program or includible in other applications. It can be accessed using: diff --git a/doc/api/stream.md b/doc/api/stream.md index e4d79623f2724e..d4f7029dd8b422 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + A stream is an abstract interface for working with streaming data in Node.js. The `stream` module provides an API for implementing the stream interface. diff --git a/doc/api/string_decoder.md b/doc/api/string_decoder.md index 5cd6121d50ddb9..59056b861321b4 100644 --- a/doc/api/string_decoder.md +++ b/doc/api/string_decoder.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `string_decoder` module provides an API for decoding `Buffer` objects into strings in a manner that preserves encoded multi-byte UTF-8 and UTF-16 characters. It can be accessed using: diff --git a/doc/api/timers.md b/doc/api/timers.md index 8b080b16ffb70e..1157ae286e0d0a 100644 --- a/doc/api/timers.md +++ b/doc/api/timers.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `timer` module exposes a global API for scheduling functions to be called at some future period of time. Because the timer functions are globals, there is no need to call `require('timers')` to use the API. diff --git a/doc/api/tls.md b/doc/api/tls.md index 41f99c91c852b0..65b35cffe96101 100644 --- a/doc/api/tls.md +++ b/doc/api/tls.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `tls` module provides an implementation of the Transport Layer Security (TLS) and Secure Socket Layer (SSL) protocols that is built on top of OpenSSL. The module can be accessed using: diff --git a/doc/api/tracing.md b/doc/api/tracing.md index 907fc2bbe23ba7..9350e0b9ca97da 100644 --- a/doc/api/tracing.md +++ b/doc/api/tracing.md @@ -4,6 +4,8 @@ > Stability: 1 - Experimental + + The `trace_events` module provides a mechanism to centralize tracing information generated by V8, Node.js core, and userspace code. diff --git a/doc/api/tty.md b/doc/api/tty.md index cd6ff1a7310db6..7dd3a9c3579477 100644 --- a/doc/api/tty.md +++ b/doc/api/tty.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `tty` module provides the `tty.ReadStream` and `tty.WriteStream` classes. In most cases, it will not be necessary or possible to use this module directly. However, it can be accessed using: diff --git a/doc/api/url.md b/doc/api/url.md index fbe8aa397918ba..06b80a438ad748 100644 --- a/doc/api/url.md +++ b/doc/api/url.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `url` module provides utilities for URL resolution and parsing. It can be accessed using: diff --git a/doc/api/util.md b/doc/api/util.md index 85a4a5782e1952..e0860da5512de8 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `util` module supports the needs of Node.js internal APIs. Many of the utilities are useful for application and module developers as well. To access it: diff --git a/doc/api/v8.md b/doc/api/v8.md index 531612330f061b..921c4378199496 100644 --- a/doc/api/v8.md +++ b/doc/api/v8.md @@ -2,6 +2,8 @@ + + The `v8` module exposes APIs that are specific to the version of [V8][] built into the Node.js binary. It can be accessed using: diff --git a/doc/api/vm.md b/doc/api/vm.md index 09250cf64cecdf..6703921c712e6a 100644 --- a/doc/api/vm.md +++ b/doc/api/vm.md @@ -6,6 +6,8 @@ + + The `vm` module enables compiling and running code within V8 Virtual Machine contexts. **The `vm` module is not a security mechanism. Do not use it to run untrusted code**. diff --git a/doc/api/wasi.md b/doc/api/wasi.md index 95e212e3c2974b..3c46a7e4f92798 100644 --- a/doc/api/wasi.md +++ b/doc/api/wasi.md @@ -4,6 +4,8 @@ > Stability: 1 - Experimental + + The WASI API provides an implementation of the [WebAssembly System Interface][] specification. WASI gives sandboxed WebAssembly applications access to the underlying operating system via a collection of POSIX-like functions. diff --git a/doc/api/worker_threads.md b/doc/api/worker_threads.md index 5d7708391d21c1..0f1dcb5ca6f139 100644 --- a/doc/api/worker_threads.md +++ b/doc/api/worker_threads.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `worker_threads` module enables the use of threads that execute JavaScript in parallel. To access it: diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 1abd3f93e9581b..759ad4f58194c2 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -4,6 +4,8 @@ > Stability: 2 - Stable + + The `zlib` module provides compression functionality implemented using Gzip, Deflate/Inflate, and Brotli. diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js index 0ffa9f9149085d..30221a7fe18b1a 100644 --- a/test/doctool/test-doctool-html.js +++ b/test/doctool/test-doctool-html.js @@ -41,7 +41,7 @@ function toHTML({ input, filename, nodeVersion, versions }) { .use(replaceLinks, { filename, linksMapper: testLinksMapper }) .use(markdown) .use(html.firstHeader) - .use(html.preprocessText) + .use(html.preprocessText, { nodeVersion }) .use(html.preprocessElements, { filename }) .use(html.buildToc, { filename, apilinks: {} }) .use(remark2rehype, { allowDangerousHTML: true }) diff --git a/tools/doc/common.js b/tools/doc/common.js index 86daae6cfc6d56..11e4ad6e2c99aa 100644 --- a/tools/doc/common.js +++ b/tools/doc/common.js @@ -7,6 +7,10 @@ function isYAMLBlock(text) { return /^/.test(text); +} + function arrify(value) { return Array.isArray(value) ? value : [value]; } @@ -43,4 +47,4 @@ function extractAndParseYAML(text) { return meta; } -module.exports = { arrify, isYAMLBlock, extractAndParseYAML }; +module.exports = { arrify, isYAMLBlock, isSourceLink, extractAndParseYAML }; diff --git a/tools/doc/generate.js b/tools/doc/generate.js index dcb72a99d931f6..93c3b263009bb6 100644 --- a/tools/doc/generate.js +++ b/tools/doc/generate.js @@ -82,7 +82,7 @@ async function main() { const content = await unified() .use(replaceLinks, { filename, linksMapper }) .use(markdown) - .use(html.preprocessText) + .use(html.preprocessText, { nodeVersion }) .use(json.jsonAPI, { filename }) .use(html.firstHeader) .use(html.preprocessElements, { filename }) diff --git a/tools/doc/html.js b/tools/doc/html.js index b58563045def8d..b8333f8bcbc498 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -104,10 +104,13 @@ function firstHeader() { // Handle general body-text replacements. // For example, link man page references to the actual page. -function preprocessText() { +function preprocessText({ nodeVersion }) { return (tree) => { visit(tree, null, (node) => { - if (node.type === 'text' && node.value) { + if (common.isSourceLink(node.value)) { + const [path] = node.value.match(/(?<=)/); + node.value = `

Source Code: ${path}

`; + } else if (node.type === 'text' && node.value) { const value = linkJsTypeDocs(linkManPages(node.value)); if (value !== node.value) { node.type = 'html'; From 66cd7bf69d6659af86cb7a140fbcb483a03f1b54 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Wed, 10 Jun 2020 14:01:02 -0700 Subject: [PATCH 018/492] doc: clarify require/import mutual exclusivity PR-URL: https://github.com/nodejs/node/pull/33832 Reviewed-By: Jan Krems Reviewed-By: Geoffrey Booth --- doc/api/esm.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/api/esm.md b/doc/api/esm.md index d5c1a66bccc0c3..bb9590b1a6c3ec 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -424,8 +424,10 @@ Node.js supports the following conditions: * `"import"` - matched when the package is loaded via `import` or `import()`. Can reference either an ES module or CommonJS file, as both `import` and `import()` can load either ES module or CommonJS sources. + _Always matched when the `"require"` condition is not matched._ * `"require"` - matched when the package is loaded via `require()`. As `require()` only supports CommonJS, the referenced file must be CommonJS. + _Always matched when the `"import"` condition is not matched._ * `"node"` - matched for any Node.js environment. Can be a CommonJS or ES module file. _This condition should always come after `"import"` or `"require"`._ From 21b0132eec7d04732982f8fc805cddfbb20cbda5 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Thu, 25 Jun 2020 21:20:23 -0700 Subject: [PATCH 019/492] doc: improve paragraph in esm.md Edit for clarity, correct tense, and brevity. PR-URL: https://github.com/nodejs/node/pull/34064 Reviewed-By: Guy Bedford Reviewed-By: Luigi Pinca --- doc/api/esm.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index bb9590b1a6c3ec..c8404a9f6892e6 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -124,12 +124,12 @@ as ES modules and `.cjs` files are always treated as CommonJS. ### Package scope and file extensions A folder containing a `package.json` file, and all subfolders below that folder -down until the next folder containing another `package.json`, is considered a -_package scope_. The `"type"` field defines how `.js` files should be treated -within a particular `package.json` file’s package scope. Every package in a +until the next folder containing another `package.json`, are a +_package scope_. The `"type"` field defines how to treat `.js` files +within the package scope. Every package in a project’s `node_modules` folder contains its own `package.json` file, so each -project’s dependencies have their own package scopes. A `package.json` lacking a -`"type"` field is treated as if it contained `"type": "commonjs"`. +project’s dependencies have their own package scopes. If a `package.json` file +does not have a `"type"` field, the default `"type"` is `"commonjs"`. The package scope applies not only to initial entry points (`node my-app.js`) but also to files referenced by `import` statements and `import()` expressions. From c6ea3d6616b89ee0e7f3e82874183831eb576272 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Thu, 25 Jun 2020 21:00:24 -0700 Subject: [PATCH 020/492] doc: make minor improvements to paragraph in child_process.md Use shorter and more direct phrasing. PR-URL: https://github.com/nodejs/node/pull/34063 Reviewed-By: Anna Henningsen Reviewed-By: Trivikram Kamat Reviewed-By: Denys Otrishko --- doc/api/child_process.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/child_process.md b/doc/api/child_process.md index 9b816414bc6659..ad9d42d9c74452 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -194,9 +194,9 @@ metacharacters may be used to trigger arbitrary command execution.** If a `callback` function is provided, it is called with the arguments `(error, stdout, stderr)`. On success, `error` will be `null`. On error, `error` will be an instance of [`Error`][]. The `error.code` property will be -the exit code of the child process while `error.signal` will be set to the -signal that terminated the process. Any exit code other than `0` is considered -to be an error. +the exit code of the process. By convention, any exit code other than `0` +indicates an error. `error.signal` will be the signal that terminated the +process. The `stdout` and `stderr` arguments passed to the callback will contain the stdout and stderr output of the child process. By default, Node.js will decode From 2e955504763c907afb0e9f220f10b28322da9c2f Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Wed, 24 Jun 2020 19:45:03 -0500 Subject: [PATCH 021/492] wasi: add reactor support PR-URL: https://github.com/nodejs/node/pull/34046 Reviewed-By: Guy Bedford Reviewed-By: Colin Ihrig --- doc/api/wasi.md | 17 ++ lib/wasi.js | 90 ++++++--- test/wasi/test-wasi-initialize-validation.js | 195 +++++++++++++++++++ test/wasi/test-wasi-start-validation.js | 9 +- 4 files changed, 278 insertions(+), 33 deletions(-) create mode 100644 test/wasi/test-wasi-initialize-validation.js diff --git a/doc/api/wasi.md b/doc/api/wasi.md index 3c46a7e4f92798..6bac95bbccf695 100644 --- a/doc/api/wasi.md +++ b/doc/api/wasi.md @@ -126,6 +126,23 @@ Attempt to begin execution of `instance` as a WASI command by invoking its If `start()` is called more than once, an exception is thrown. +### `wasi.initialize(instance)` + + +* `instance` {WebAssembly.Instance} + +Attempt to initialize `instance` as a WASI reactor by invoking its +`_initialize()` export, if it is present. If `instance` contains a `_start()` +export, then an exception is thrown. + +`initialize()` requires that `instance` exports a [`WebAssembly.Memory`][] named +`memory`. If `instance` does not have a `memory` export an exception is thrown. + +If `initialize()` is called more than once, an exception is thrown. + ### `wasi.wasiImport` * {Object} @@ -1791,7 +1791,7 @@ added: v11.12.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * {string} @@ -1810,7 +1810,7 @@ added: v11.12.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * {string} @@ -1829,7 +1829,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * `err` {Error} A custom error used for reporting the JavaScript stack. @@ -1872,7 +1872,7 @@ added: v11.12.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * {boolean} @@ -1890,7 +1890,7 @@ added: v11.12.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * {boolean} @@ -1907,7 +1907,7 @@ added: v11.12.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * {string} @@ -1925,7 +1925,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This API is no longer considered experimental. + description: This API is no longer experimental. --> * `filename` {string} Name of the file where the report is written. This From 0672384be9bfc211256e5c17304baac557779f15 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Sun, 28 Jun 2020 21:08:22 -0700 Subject: [PATCH 028/492] doc: change "currently not considered public" to "not supported" PR-URL: https://github.com/nodejs/node/pull/34114 Reviewed-By: Trivikram Kamat Reviewed-By: Anna Henningsen --- doc/api/async_hooks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/async_hooks.md b/doc/api/async_hooks.md index 67a475a25b359b..de83dacaf46738 100644 --- a/doc/api/async_hooks.md +++ b/doc/api/async_hooks.md @@ -304,7 +304,7 @@ been initialized. This can contain useful information that can vary based on the value of `type`. For instance, for the `GETADDRINFOREQWRAP` resource type, `resource` provides the host name used when looking up the IP address for the host in `net.Server.listen()`. The API for accessing this information is -currently not considered public, but using the Embedder API, users can provide +not supported, but using the Embedder API, users can provide and document their own resource objects. For example, such a resource object could contain the SQL query being executed. From 68654da30d38c292221d98815f2e184d6b14d2c0 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Sun, 28 Jun 2020 20:32:12 +0200 Subject: [PATCH 029/492] tls: remove unnecessary close listener Wrapped streams are expected to behave the same as socket with handle. Remove unnecessary difference in handling. PR-URL: https://github.com/nodejs/node/pull/34105 Reviewed-By: Anna Henningsen Reviewed-By: Trivikram Kamat --- lib/_tls_wrap.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/_tls_wrap.js b/lib/_tls_wrap.js index 2a596b67329098..546f8f5b89ad35 100644 --- a/lib/_tls_wrap.js +++ b/lib/_tls_wrap.js @@ -487,7 +487,6 @@ function TLSSocket(socket, opts) { // handle, but a JS stream doesn't have one. Wrap it up to make it look like // a socket. wrap = new JSStreamSocket(socket); - wrap.once('close', () => this.destroy()); } // Just a documented property to make secure sockets From 6c739aac5585ddeaefa48d70057c908ea20e6a5e Mon Sep 17 00:00:00 2001 From: falguniraina <48027052+falguniraina@users.noreply.github.com> Date: Fri, 19 Jun 2020 21:28:52 +0530 Subject: [PATCH 030/492] doc: improve text in issues.md PR-URL: https://github.com/nodejs/node/pull/33973 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Rich Trott Reviewed-By: Trivikram Kamat Reviewed-By: Myles Borins Reviewed-By: Zeyu Yang --- doc/guides/contributing/issues.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/guides/contributing/issues.md b/doc/guides/contributing/issues.md index 31a47c1cd33c16..a021e4c8207c7d 100644 --- a/doc/guides/contributing/issues.md +++ b/doc/guides/contributing/issues.md @@ -90,10 +90,10 @@ including whether the behavior being seen is a bug or a feature. This discussion is part of the process and should be kept focused, helpful, and professional. Short, clipped responses that provide neither additional context nor supporting -detail are not helpful or professional. To many, such responses are simply +details are not helpful or professional to many. Such responses are simply annoying and unfriendly. -Contributors are encouraged to help one another make forward progress as much +Contributors are encouraged to help one another to make forward progress as much as possible, empowering one another to solve issues collaboratively. If you choose to comment on an issue that you feel either is not a problem that needs to be fixed, or if you encounter information in an issue that you feel is From 3d5f7674e7048d707c3587f84e95138423910626 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Mon, 29 Jun 2020 21:17:05 -0700 Subject: [PATCH 031/492] doc: changed "considered experimental" to "experimental" in cli.md Change "no longer considered experimental" to "no longer experimental" in cli.md. PR-URL: https://github.com/nodejs/node/pull/34128 Reviewed-By: Denys Otrishko Reviewed-By: Trivikram Kamat --- doc/api/cli.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/api/cli.md b/doc/api/cli.md index 27ab46d5553ac5..d6bf099b8ef2b9 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -638,7 +638,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: Changed from `--diagnostic-report-directory` to @@ -653,7 +653,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: changed from `--diagnostic-report-filename` to @@ -668,7 +668,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32496 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: changed from `--diagnostic-report-on-fatalerror` to @@ -687,7 +687,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: changed from `--diagnostic-report-on-signal` to @@ -704,7 +704,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: changed from `--diagnostic-report-signal` to @@ -720,7 +720,7 @@ added: v11.8.0 changes: - version: v12.17.0 pr-url: https://github.com/nodejs/node/pull/32242 - description: This option is no longer considered experimental. + description: This option is no longer experimental. - version: v12.0.0 pr-url: https://github.com/nodejs/node/pull/27312 description: changed from `--diagnostic-report-uncaught-exception` to From c78ef2d35cb94bf2b2453ab1e404bfc893611d9f Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Mon, 29 Jun 2020 21:21:19 -0700 Subject: [PATCH 032/492] doc: change "considered experimental" to "experimental" in n-api.md This changes "considered" experimental" to "experimental" in the n-api.md file and introduces some additional brevity. PR-URL: https://github.com/nodejs/node/pull/34129 Reviewed-By: Denys Otrishko Reviewed-By: Trivikram Kamat --- doc/api/n-api.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/api/n-api.md b/doc/api/n-api.md index 207f840257ebaa..7002dfef61baa3 100644 --- a/doc/api/n-api.md +++ b/doc/api/n-api.md @@ -222,8 +222,7 @@ can be specified explicitly when including the header: This restricts the N-API surface to just the functionality that was available in the specified (and earlier) versions. -Some of the N-API surface is considered experimental and requires explicit -opt-in to access those APIs: +Some of the N-API surface is experimental and requires explicit opt-in: ```c #define NAPI_EXPERIMENTAL From 19bfc012d136a0cb6a70a3e2715ede12c0b1dda8 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Sat, 27 Jun 2020 21:29:58 -0700 Subject: [PATCH 033/492] doc: move sam-github to TSC Emeriti I don't have enough time to remain active in the TSC, so I will step down. PR-URL: https://github.com/nodejs/node/pull/34095 Reviewed-By: Gireesh Punathil Reviewed-By: Daniel Bevenius Reviewed-By: Colin Ihrig --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fda0fc98ea2340..3387975cca122c 100644 --- a/README.md +++ b/README.md @@ -187,8 +187,6 @@ For information about the governance of the Node.js project, see **Matheus Marchini** <mat@mmarchini.me> * [MylesBorins](https://github.com/MylesBorins) - **Myles Borins** <myles.borins@gmail.com> (he/him) -* [sam-github](https://github.com/sam-github) - -**Sam Roberts** <vieuxtech@gmail.com> * [targos](https://github.com/targos) - **Michaël Zasso** <targos@protonmail.com> (he/him) * [tniessen](https://github.com/tniessen) - @@ -226,6 +224,8 @@ For information about the governance of the Node.js project, see **Bert Belder** <bertbelder@gmail.com> * [rvagg](https://github.com/rvagg) - **Rod Vagg** <r@va.gg> +* [sam-github](https://github.com/sam-github) - +**Sam Roberts** <vieuxtech@gmail.com> * [shigeki](https://github.com/shigeki) - **Shigeki Ohtsu** <ohtsu@ohtsu.org> (he/him) * [thefourtheye](https://github.com/thefourtheye) - @@ -391,8 +391,6 @@ For information about the governance of the Node.js project, see **Ujjwal Sharma** <ryzokuken@disroot.org> (he/him) * [saghul](https://github.com/saghul) - **Saúl Ibarra Corretgé** <saghul@gmail.com> -* [sam-github](https://github.com/sam-github) - -**Sam Roberts** <vieuxtech@gmail.com> * [santigimeno](https://github.com/santigimeno) - **Santiago Gimeno** <santiago.gimeno@gmail.com> * [sebdeckers](https://github.com/sebdeckers) - @@ -530,6 +528,8 @@ For information about the governance of the Node.js project, see **Roman Klauke** <romaaan.git@gmail.com> * [RReverser](https://github.com/RReverser) - **Ingvar Stepanyan** <me@rreverser.com> +* [sam-github](https://github.com/sam-github) - +**Sam Roberts** <vieuxtech@gmail.com> * [stefanmb](https://github.com/stefanmb) - **Stefan Budeanu** <stefan@budeanu.com> * [tellnes](https://github.com/tellnes) - From e2fff1b1b0b9911071f1bddbc27238678b843ef5 Mon Sep 17 00:00:00 2001 From: Derek Lewis Date: Sun, 7 Jun 2020 17:21:45 -0400 Subject: [PATCH 034/492] doc: add http highlight grammar Prior to this commit, http request message code blocks in Markdown files were not being highlighted correctly. This has been corrected by adding the new grammar to the bundle, removing the CRLFs (`\r\n`) from these code samples, adding a reminder to re-add them, and tuning the syntax theme to support attribute highlighting. PR-URL: https://github.com/nodejs/node/pull/33785 Reviewed-By: Rich Trott Reviewed-By: Luigi Pinca Reviewed-By: Matteo Collina --- doc/api/http.md | 10 +++++----- doc/api/http2.md | 10 +++++----- doc/api_assets/README.md | 3 ++- doc/api_assets/highlight.pack.js | 2 +- doc/api_assets/hljs.css | 1 + 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/doc/api/http.md b/doc/api/http.md index f372831a95a9f1..1ae77464087e2c 100644 --- a/doc/api/http.md +++ b/doc/api/http.md @@ -1982,13 +1982,13 @@ added: v0.1.90 **Only valid for request obtained from [`http.Server`][].** -Request URL string. This contains only the URL that is -present in the actual HTTP request. If the request is: +Request URL string. This contains only the URL that is present in the actual +HTTP request. If the request is (remember, each header line ends with `\r\n` +plus a final `\r\n` after the last one indicating the end of the header): ```http -GET /status?name=ryan HTTP/1.1\r\n -Accept: text/plain\r\n -\r\n +GET /status?name=ryan HTTP/1.1 +Accept: text/plain ``` To parse the URL into its parts: diff --git a/doc/api/http2.md b/doc/api/http2.md index 779a5b69e44c46..3c4c7b85a79f54 100644 --- a/doc/api/http2.md +++ b/doc/api/http2.md @@ -2987,13 +2987,13 @@ added: v8.4.0 * {string} -Request URL string. This contains only the URL that is -present in the actual HTTP request. If the request is: +Request URL string. This contains only the URL that is present in the actual +HTTP request. If the request is (remember, each header line ends with `\r\n` +plus a final `\r\n` after the last one indicating the end of the header): ```http -GET /status?name=ryan HTTP/1.1\r\n -Accept: text/plain\r\n -\r\n +GET /status?name=ryan HTTP/1.1 +Accept: text/plain ``` Then `request.url` will be: diff --git a/doc/api_assets/README.md b/doc/api_assets/README.md index 58799cf12fcb88..3c0e1d1cbb2976 100644 --- a/doc/api_assets/README.md +++ b/doc/api_assets/README.md @@ -2,7 +2,7 @@ ## highlight.pack.js -_Generated by [highlightjs.org/download][] on 2020-05-16._ +_Generated by [highlightjs.org/download][] on 2020-06-07._ Grammars included in the custom bundle: @@ -10,6 +10,7 @@ Grammars included in the custom bundle: * C * C++ * CoffeeScript +* HTTP * JavaScript * JSON * Markdown diff --git a/doc/api_assets/highlight.pack.js b/doc/api_assets/highlight.pack.js index a493e16d3a7238..bd134963d24609 100644 --- a/doc/api_assets/highlight.pack.js +++ b/doc/api_assets/highlight.pack.js @@ -3,4 +3,4 @@ License: BSD-3-Clause Copyright (c) 2006-2020, Ivan Sagalaev */ -var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!n.hasOwnProperty(r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}function n(e){return e.replace(/&/g,"&").replace(//g,">")}function t(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach((function(e){for(n in e)t[n]=e[n]})),t}function r(e){return e.nodeName.toLowerCase()}var a=Object.freeze({__proto__:null,escapeHTML:n,inherit:t,nodeStream:function(e){var n=[];return function e(t,a){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=e(i,a),r(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n},mergeStreams:function(e,t,a){var i=0,s="",o=[];function l(){return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||t.length;){var g=l();if(s+=n(a.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+n(a.substr(i))}});const i="",s=e=>!!e.kind;class o{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=n(e)}openNode(e){if(!s(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){s(e)&&(this.buffer+=i)}span(e){this.buffer+=``}value(){return this.buffer}}class l{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){let n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){e.children&&(e.children.every(e=>"string"==typeof e)?(e.text=e.children.join(""),delete e.children):e.children.forEach(e=>{"string"!=typeof e&&l._collapse(e)}))}}class c extends l{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){let t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new o(this,this.options).value()}finalize(){}}function u(e){return e&&e.source||e}const d="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",g={begin:"\\\\[\\s\\S]",relevance:0},h={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[g]},f={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[g]},p={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,r){var a=t({className:"comment",begin:e,end:n,contains:[]},r||{});return a.contains.push(p),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|XXX):",relevance:0}),a},b=m("//","$"),v=m("/\\*","\\*/"),x=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:d,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",BACKSLASH_ESCAPE:g,APOS_STRING_MODE:h,QUOTE_STRING_MODE:f,PHRASAL_WORDS_MODE:p,COMMENT:m,C_LINE_COMMENT_MODE:b,C_BLOCK_COMMENT_MODE:v,HASH_COMMENT_MODE:x,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:d,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^\/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[g,{begin:/\[/,end:/\]/,relevance:0,contains:[g]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0}}),E="of and for in not or if then".split(" ");function R(e,n){return n?+n:(t=e,E.includes(t.toLowerCase())?0:1);var t}const N=n,w=t,{nodeStream:y,mergeStreams:O}=a;return function(n){var r=[],a={},i={},s=[],o=!0,l=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,d="Could not find the language '{}', did you forget to load/include a language module?",g={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0,__emitter:c};function h(e){return g.noHighlightRe.test(e)}function f(e,n,t,r){var a={code:n,language:e};T("before:highlight",a);var i=a.result?a.result:p(a.language,a.code,t,r);return i.code=a.code,T("after:highlight",i),i}function p(e,n,r,i){var s=n;function l(e,n){var t=v.case_insensitive?n[0].toLowerCase():n[0];return e.keywords.hasOwnProperty(t)&&e.keywords[t]}function c(){null!=_.subLanguage?function(){if(""!==k){var e="string"==typeof _.subLanguage;if(!e||a[_.subLanguage]){var n=e?p(_.subLanguage,k,!0,E[_.subLanguage]):m(k,_.subLanguage.length?_.subLanguage:void 0);_.relevance>0&&(T+=n.relevance),e&&(E[_.subLanguage]=n.top),w.addSublanguage(n.emitter,n.language)}else w.addText(k)}}():function(){var e,n,t,r;if(_.keywords){for(n=0,_.lexemesRe.lastIndex=0,t=_.lexemesRe.exec(k),r="";t;){r+=k.substring(n,t.index);var a=null;(e=l(_,t))?(w.addText(r),r="",T+=e[1],a=e[0],w.addKeyword(t[0],a)):r+=t[0],n=_.lexemesRe.lastIndex,t=_.lexemesRe.exec(k)}r+=k.substr(n),w.addText(r)}else w.addText(k)}(),k=""}function h(e){e.className&&w.openNode(e.className),_=Object.create(e,{parent:{value:_}})}var f={};function b(n,t){var a,i=t&&t[0];if(k+=n,null==i)return c(),0;if("begin"==f.type&&"end"==t.type&&f.index==t.index&&""===i){if(k+=s.slice(t.index,t.index+1),!o)throw(a=Error("0 width match regex")).languageName=e,a.badRule=f.rule,a;return 1}if(f=t,"begin"===t.type)return function(e){var n=e[0],t=e.rule;return t.__onBegin&&(t.__onBegin(e)||{}).ignoreMatch?function(e){return 0===_.matcher.regexIndex?(k+=e[0],1):(B=!0,0)}(n):(t&&t.endSameAsBegin&&(t.endRe=RegExp(n.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),t.skip?k+=n:(t.excludeBegin&&(k+=n),c(),t.returnBegin||t.excludeBegin||(k=n)),h(t),t.returnBegin?0:n.length)}(t);if("illegal"===t.type&&!r)throw(a=Error('Illegal lexeme "'+i+'" for mode "'+(_.className||"")+'"')).mode=_,a;if("end"===t.type){var l=function(e){var n=e[0],t=s.substr(e.index),r=function e(n,t){if(function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(n.endRe,t)){for(;n.endsParent&&n.parent;)n=n.parent;return n}if(n.endsWithParent)return e(n.parent,t)}(_,t);if(r){var a=_;a.skip?k+=n:(a.returnEnd||a.excludeEnd||(k+=n),c(),a.excludeEnd&&(k=n));do{_.className&&w.closeNode(),_.skip||_.subLanguage||(T+=_.relevance),_=_.parent}while(_!==r.parent);return r.starts&&(r.endSameAsBegin&&(r.starts.endRe=r.endRe),h(r.starts)),a.returnEnd?0:n.length}}(t);if(null!=l)return l}if("illegal"===t.type&&""===i)return 1;if(A>1e5&&A>3*t.index)throw Error("potential infinite loop, way more iterations than matches");return k+=i,i.length}var v=M(e);if(!v)throw console.error(d.replace("{}",e)),Error('Unknown language: "'+e+'"');!function(e){function n(n,t){return RegExp(u(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class r{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);let e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+="|"),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"==l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("==l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;let n=this.matcherRe.exec(e);if(!n)return null;let t=n.findIndex((e,n)=>n>0&&null!=e),r=this.matchIndexes[t];return Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];let n=new r;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){let n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;let t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e){let n=e.input[e.index-1],t=e.input[e.index+e[0].length];if("."===n||"."===t)return{ignoreMatch:!0}}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");!function r(s,o){s.compiled||(s.compiled=!0,s.__onBegin=null,s.keywords=s.keywords||s.beginKeywords,s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,R(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemesRe=n(s.lexemes||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__onBegin=i),s.begin||(s.begin=/\B|\b/),s.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(s.endRe=n(s.end)),s.terminator_end=u(s.end)||"",s.endsWithParent&&o.terminator_end&&(s.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(s.illegalRe=n(s.illegal)),null==s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return t(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?t(e,{starts:e.starts?t(e.starts):null}):Object.isFrozen(e)?t(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){r(e,s)})),s.starts&&r(s.starts,o),s.matcher=function(e){let n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(s))}(e)}(v);var x,_=i||v,E={},w=new g.__emitter(g);!function(){for(var e=[],n=_;n!==v;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>w.openNode(e))}();var y,O,k="",T=0,L=0,A=0,B=!1;try{for(_.matcher.considerAll();A++,B?B=!1:(_.matcher.lastIndex=L,_.matcher.considerAll()),y=_.matcher.exec(s);)O=b(s.substring(L,y.index),y),L=y.index+O;return b(s.substr(L)),w.closeAllNodes(),w.finalize(),x=w.toHTML(),{relevance:T,value:x,language:e,illegal:!1,emitter:w,top:_}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:s.slice(L-100,L+100),mode:n.mode},sofar:x,relevance:0,value:N(s),emitter:w};if(o)return{relevance:0,value:N(s),emitter:w,language:e,top:_,errorRaised:n};throw n}}function m(e,n){n=n||g.languages||Object.keys(a);var t=function(e){const n={relevance:0,emitter:new g.__emitter(g),value:N(e),illegal:!1,top:E};return n.emitter.addText(e),n}(e),r=t;return n.filter(M).filter(k).forEach((function(n){var a=p(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function b(e){return g.tabReplace||g.useBR?e.replace(l,(function(e,n){return g.useBR&&"\n"===e?"
":g.tabReplace?n.replace(/\t/g,g.tabReplace):""})):e}function v(e){var n,t,r,a,s,o=function(e){var n,t=e.className+" ";if(t+=e.parentNode?e.parentNode.className:"",n=g.languageDetectRe.exec(t)){var r=M(n[1]);return r||(console.warn(d.replace("{}",n[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?n[1]:"no-highlight"}return t.split(/\s+/).find(e=>h(e)||M(e))}(e);h(o)||(T("before:highlightBlock",{block:e,language:o}),g.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e,s=n.textContent,r=o?f(o,s,!0):m(s),(t=y(n)).length&&((a=document.createElement("div")).innerHTML=r.value,r.value=O(t,y(a),s)),r.value=b(r.value),T("after:highlightBlock",{block:e,result:r}),e.innerHTML=r.value,e.className=function(e,n,t){var r=n?i[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,o,r.language),e.result={language:r.language,re:r.relevance},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.relevance}))}function x(){if(!x.called){x.called=!0;var e=document.querySelectorAll("pre code");r.forEach.call(e,v)}}const E={disableAutodetect:!0,name:"Plain text"};function M(e){return e=(e||"").toLowerCase(),a[e]||a[i[e]]}function k(e){var n=M(e);return n&&!n.disableAutodetect}function T(e,n){var t=e;s.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(n,{highlight:f,highlightAuto:m,fixMarkup:b,highlightBlock:v,configure:function(e){g=w(g,e)},initHighlighting:x,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",x,!1)},registerLanguage:function(e,t){var r;try{r=t(n)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!o)throw n;console.error(n),r=E}r.name||(r.name=e),a[e]=r,r.rawDefinition=t.bind(null,n),r.aliases&&r.aliases.forEach((function(n){i[n]=e}))},listLanguages:function(){return Object.keys(a)},getLanguage:M,requireLanguage:function(e){var n=M(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:k,inherit:w,addPlugin:function(e,n){s.push(e)}}),n.debugMode=function(){o=!1},n.safeMode=function(){o=!0},n.versionString="10.0.3";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(n,_),n}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}());hljs.registerLanguage("xml",function(){"use strict";return function(e){var n={className:"symbol",begin:"&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;"},a={begin:"\\s",contains:[{className:"meta-keyword",begin:"#?[a-z_][a-z1-9_-]+",illegal:"\\n"}]},s=e.inherit(a,{begin:"\\(",end:"\\)"}),t=e.inherit(e.APOS_STRING_MODE,{className:"meta-string"}),i=e.inherit(e.QUOTE_STRING_MODE,{className:"meta-string"}),c={endsWithParent:!0,illegal:/`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}());hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}());hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const n={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},t={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,n]};n.contains.push(t);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]};return{name:"Bash",aliases:["sh","zsh"],lexemes:/\b-?[a-z\._]+\b/,keywords:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[{className:"meta",begin:/^#![^\n]+sh\s*$/,relevance:10},{className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0},a,e.HASH_COMMENT_MODE,t,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}());hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}());hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},{begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\((?:.|\n)*?\)\1"/}]},s={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},i={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},o=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+o,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:o,returnBegin:!0,contains:[c],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:i,strings:a,keywords:l}}}}());hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}());hljs.registerLanguage("coffeescript",function(){"use strict";return function(e){var n={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger super yield import export from as default await then unless until loop of by when and or is isnt not",literal:"true false null undefined yes no on off",built_in:"npm require console print module global window document"},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:n},a=[e.BINARY_NUMBER_MODE,e.inherit(e.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[e.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,e.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=a;var t=e.inherit(e.TITLE_MODE,{begin:i}),r={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:n,contains:["self"].concat(a)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:n,illegal:/\/\*/,contains:a.concat([e.COMMENT("###","###"),e.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[t,r]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[r]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[t]},t]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}());hljs.registerLanguage("javascript",function(){"use strict";return function(e){var n={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},a="[A-Za-z$_][0-9A-Za-z$_]*",s={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},r={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:e.C_NUMBER_RE+"n?"}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:s,contains:[]},t={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"xml"}},c={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"css"}},o={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,r,e.REGEXP_MODE];var l=i.contains.concat([e.C_BLOCK_COMMENT_MODE,e.C_LINE_COMMENT_MODE]),d={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:l};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:s,contains:[{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},{className:"meta",begin:/^#!/,end:/$/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,e.C_LINE_COMMENT_MODE,e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:a+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),e.C_BLOCK_COMMENT_MODE,r,{begin:/[{,\n]\s*/,relevance:0,contains:[{begin:a+"\\s*:",returnBegin:!0,relevance:0,contains:[{className:"attr",begin:a,relevance:0}]}]},{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+a+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:a},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:s,contains:l}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:n.begin,end:n.end}],subLanguage:"xml",contains:[{begin:n.begin,end:n.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:a}),d],illegal:/\[|%/},{begin:/\$[(.]/},e.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+a+"\\()",end:/{/,keywords:"get set",contains:[e.inherit(e.TITLE_MODE,{begin:a}),{begin:/\(\)/},d]}],illegal:/#(?!!)/}}}());hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}());hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}()); \ No newline at end of file +var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!n.hasOwnProperty(r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}function n(e){return e.replace(/&/g,"&").replace(//g,">")}function t(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach((function(e){for(n in e)t[n]=e[n]})),t}function r(e){return e.nodeName.toLowerCase()}var a=Object.freeze({__proto__:null,escapeHTML:n,inherit:t,nodeStream:function(e){var n=[];return function e(t,a){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=e(i,a),r(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n},mergeStreams:function(e,t,a){var i=0,s="",o=[];function l(){return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||t.length;){var g=l();if(s+=n(a.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+n(a.substr(i))}});const i="",s=e=>!!e.kind;class o{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=n(e)}openNode(e){if(!s(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){s(e)&&(this.buffer+=i)}span(e){this.buffer+=``}value(){return this.buffer}}class l{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){let n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){e.children&&(e.children.every(e=>"string"==typeof e)?(e.text=e.children.join(""),delete e.children):e.children.forEach(e=>{"string"!=typeof e&&l._collapse(e)}))}}class c extends l{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){let t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new o(this,this.options).value()}finalize(){}}function u(e){return e&&e.source||e}const d="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",g={begin:"\\\\[\\s\\S]",relevance:0},h={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[g]},f={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[g]},p={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,r){var a=t({className:"comment",begin:e,end:n,contains:[]},r||{});return a.contains.push(p),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|XXX):",relevance:0}),a},b=m("//","$"),v=m("/\\*","\\*/"),x=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:d,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",BACKSLASH_ESCAPE:g,APOS_STRING_MODE:h,QUOTE_STRING_MODE:f,PHRASAL_WORDS_MODE:p,COMMENT:m,C_LINE_COMMENT_MODE:b,C_BLOCK_COMMENT_MODE:v,HASH_COMMENT_MODE:x,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:d,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^\/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[g,{begin:/\[/,end:/\]/,relevance:0,contains:[g]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0}}),E="of and for in not or if then".split(" ");function R(e,n){return n?+n:(t=e,E.includes(t.toLowerCase())?0:1);var t}const N=n,w=t,{nodeStream:y,mergeStreams:O}=a;return function(n){var r=[],a={},i={},s=[],o=!0,l=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,d="Could not find the language '{}', did you forget to load/include a language module?",g={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0,__emitter:c};function h(e){return g.noHighlightRe.test(e)}function f(e,n,t,r){var a={code:n,language:e};T("before:highlight",a);var i=a.result?a.result:p(a.language,a.code,t,r);return i.code=a.code,T("after:highlight",i),i}function p(e,n,r,i){var s=n;function l(e,n){var t=v.case_insensitive?n[0].toLowerCase():n[0];return e.keywords.hasOwnProperty(t)&&e.keywords[t]}function c(){null!=_.subLanguage?function(){if(""!==k){var e="string"==typeof _.subLanguage;if(!e||a[_.subLanguage]){var n=e?p(_.subLanguage,k,!0,E[_.subLanguage]):m(k,_.subLanguage.length?_.subLanguage:void 0);_.relevance>0&&(T+=n.relevance),e&&(E[_.subLanguage]=n.top),w.addSublanguage(n.emitter,n.language)}else w.addText(k)}}():function(){var e,n,t,r;if(_.keywords){for(n=0,_.lexemesRe.lastIndex=0,t=_.lexemesRe.exec(k),r="";t;){r+=k.substring(n,t.index);var a=null;(e=l(_,t))?(w.addText(r),r="",T+=e[1],a=e[0],w.addKeyword(t[0],a)):r+=t[0],n=_.lexemesRe.lastIndex,t=_.lexemesRe.exec(k)}r+=k.substr(n),w.addText(r)}else w.addText(k)}(),k=""}function h(e){e.className&&w.openNode(e.className),_=Object.create(e,{parent:{value:_}})}var f={};function b(n,t){var a,i=t&&t[0];if(k+=n,null==i)return c(),0;if("begin"==f.type&&"end"==t.type&&f.index==t.index&&""===i){if(k+=s.slice(t.index,t.index+1),!o)throw(a=Error("0 width match regex")).languageName=e,a.badRule=f.rule,a;return 1}if(f=t,"begin"===t.type)return function(e){var n=e[0],t=e.rule;return t.__onBegin&&(t.__onBegin(e)||{}).ignoreMatch?function(e){return 0===_.matcher.regexIndex?(k+=e[0],1):(B=!0,0)}(n):(t&&t.endSameAsBegin&&(t.endRe=RegExp(n.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),t.skip?k+=n:(t.excludeBegin&&(k+=n),c(),t.returnBegin||t.excludeBegin||(k=n)),h(t),t.returnBegin?0:n.length)}(t);if("illegal"===t.type&&!r)throw(a=Error('Illegal lexeme "'+i+'" for mode "'+(_.className||"")+'"')).mode=_,a;if("end"===t.type){var l=function(e){var n=e[0],t=s.substr(e.index),r=function e(n,t){if(function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(n.endRe,t)){for(;n.endsParent&&n.parent;)n=n.parent;return n}if(n.endsWithParent)return e(n.parent,t)}(_,t);if(r){var a=_;a.skip?k+=n:(a.returnEnd||a.excludeEnd||(k+=n),c(),a.excludeEnd&&(k=n));do{_.className&&w.closeNode(),_.skip||_.subLanguage||(T+=_.relevance),_=_.parent}while(_!==r.parent);return r.starts&&(r.endSameAsBegin&&(r.starts.endRe=r.endRe),h(r.starts)),a.returnEnd?0:n.length}}(t);if(null!=l)return l}if("illegal"===t.type&&""===i)return 1;if(A>1e5&&A>3*t.index)throw Error("potential infinite loop, way more iterations than matches");return k+=i,i.length}var v=M(e);if(!v)throw console.error(d.replace("{}",e)),Error('Unknown language: "'+e+'"');!function(e){function n(n,t){return RegExp(u(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class r{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);let e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+="|"),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"==l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("==l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;let n=this.matcherRe.exec(e);if(!n)return null;let t=n.findIndex((e,n)=>n>0&&null!=e),r=this.matchIndexes[t];return Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];let n=new r;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){let n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;let t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e){let n=e.input[e.index-1],t=e.input[e.index+e[0].length];if("."===n||"."===t)return{ignoreMatch:!0}}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");!function r(s,o){s.compiled||(s.compiled=!0,s.__onBegin=null,s.keywords=s.keywords||s.beginKeywords,s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,R(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemesRe=n(s.lexemes||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__onBegin=i),s.begin||(s.begin=/\B|\b/),s.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(s.endRe=n(s.end)),s.terminator_end=u(s.end)||"",s.endsWithParent&&o.terminator_end&&(s.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(s.illegalRe=n(s.illegal)),null==s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return t(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?t(e,{starts:e.starts?t(e.starts):null}):Object.isFrozen(e)?t(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){r(e,s)})),s.starts&&r(s.starts,o),s.matcher=function(e){let n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(s))}(e)}(v);var x,_=i||v,E={},w=new g.__emitter(g);!function(){for(var e=[],n=_;n!==v;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>w.openNode(e))}();var y,O,k="",T=0,L=0,A=0,B=!1;try{for(_.matcher.considerAll();A++,B?B=!1:(_.matcher.lastIndex=L,_.matcher.considerAll()),y=_.matcher.exec(s);)O=b(s.substring(L,y.index),y),L=y.index+O;return b(s.substr(L)),w.closeAllNodes(),w.finalize(),x=w.toHTML(),{relevance:T,value:x,language:e,illegal:!1,emitter:w,top:_}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:s.slice(L-100,L+100),mode:n.mode},sofar:x,relevance:0,value:N(s),emitter:w};if(o)return{relevance:0,value:N(s),emitter:w,language:e,top:_,errorRaised:n};throw n}}function m(e,n){n=n||g.languages||Object.keys(a);var t=function(e){const n={relevance:0,emitter:new g.__emitter(g),value:N(e),illegal:!1,top:E};return n.emitter.addText(e),n}(e),r=t;return n.filter(M).filter(k).forEach((function(n){var a=p(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function b(e){return g.tabReplace||g.useBR?e.replace(l,(function(e,n){return g.useBR&&"\n"===e?"
":g.tabReplace?n.replace(/\t/g,g.tabReplace):""})):e}function v(e){var n,t,r,a,s,o=function(e){var n,t=e.className+" ";if(t+=e.parentNode?e.parentNode.className:"",n=g.languageDetectRe.exec(t)){var r=M(n[1]);return r||(console.warn(d.replace("{}",n[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?n[1]:"no-highlight"}return t.split(/\s+/).find(e=>h(e)||M(e))}(e);h(o)||(T("before:highlightBlock",{block:e,language:o}),g.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e,s=n.textContent,r=o?f(o,s,!0):m(s),(t=y(n)).length&&((a=document.createElement("div")).innerHTML=r.value,r.value=O(t,y(a),s)),r.value=b(r.value),T("after:highlightBlock",{block:e,result:r}),e.innerHTML=r.value,e.className=function(e,n,t){var r=n?i[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,o,r.language),e.result={language:r.language,re:r.relevance},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.relevance}))}function x(){if(!x.called){x.called=!0;var e=document.querySelectorAll("pre code");r.forEach.call(e,v)}}const E={disableAutodetect:!0,name:"Plain text"};function M(e){return e=(e||"").toLowerCase(),a[e]||a[i[e]]}function k(e){var n=M(e);return n&&!n.disableAutodetect}function T(e,n){var t=e;s.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(n,{highlight:f,highlightAuto:m,fixMarkup:b,highlightBlock:v,configure:function(e){g=w(g,e)},initHighlighting:x,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",x,!1)},registerLanguage:function(e,t){var r;try{r=t(n)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!o)throw n;console.error(n),r=E}r.name||(r.name=e),a[e]=r,r.rawDefinition=t.bind(null,n),r.aliases&&r.aliases.forEach((function(n){i[n]=e}))},listLanguages:function(){return Object.keys(a)},getLanguage:M,requireLanguage:function(e){var n=M(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:k,inherit:w,addPlugin:function(e,n){s.push(e)}}),n.debugMode=function(){o=!1},n.safeMode=function(){o=!0},n.versionString="10.0.3";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(n,_),n}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},{begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\((?:.|\n)*?\)\1"/}]},s={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},i={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},o=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+o,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:o,returnBegin:!0,contains:[c],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:i,strings:a,keywords:l}}}}());hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}());hljs.registerLanguage("xml",function(){"use strict";return function(e){var n={className:"symbol",begin:"&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;"},a={begin:"\\s",contains:[{className:"meta-keyword",begin:"#?[a-z_][a-z1-9_-]+",illegal:"\\n"}]},s=e.inherit(a,{begin:"\\(",end:"\\)"}),t=e.inherit(e.APOS_STRING_MODE,{className:"meta-string"}),i=e.inherit(e.QUOTE_STRING_MODE,{className:"meta-string"}),c={endsWithParent:!0,illegal:/`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}());hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}());hljs.registerLanguage("coffeescript",function(){"use strict";return function(e){var n={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger super yield import export from as default await then unless until loop of by when and or is isnt not",literal:"true false null undefined yes no on off",built_in:"npm require console print module global window document"},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:n},a=[e.BINARY_NUMBER_MODE,e.inherit(e.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[e.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,e.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=a;var t=e.inherit(e.TITLE_MODE,{begin:i}),r={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:n,contains:["self"].concat(a)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:n,illegal:/\/\*/,contains:a.concat([e.COMMENT("###","###"),e.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[t,r]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[r]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[t]},t]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}());hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const n={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},t={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,n]};n.contains.push(t);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]};return{name:"Bash",aliases:["sh","zsh"],lexemes:/\b-?[a-z\._]+\b/,keywords:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[{className:"meta",begin:/^#![^\n]+sh\s*$/,relevance:10},{className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0},a,e.HASH_COMMENT_MODE,t,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}());hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}());hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}());hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}());hljs.registerLanguage("javascript",function(){"use strict";return function(e){var n={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},a="[A-Za-z$_][0-9A-Za-z$_]*",s={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},r={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:e.C_NUMBER_RE+"n?"}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:s,contains:[]},t={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"xml"}},c={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"css"}},o={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,r,e.REGEXP_MODE];var l=i.contains.concat([e.C_BLOCK_COMMENT_MODE,e.C_LINE_COMMENT_MODE]),d={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:l};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:s,contains:[{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},{className:"meta",begin:/^#!/,end:/$/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,e.C_LINE_COMMENT_MODE,e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:a+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),e.C_BLOCK_COMMENT_MODE,r,{begin:/[{,\n]\s*/,relevance:0,contains:[{begin:a+"\\s*:",returnBegin:!0,relevance:0,contains:[{className:"attr",begin:a,relevance:0}]}]},{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+a+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:a},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:s,contains:l}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:n.begin,end:n.end}],subLanguage:"xml",contains:[{begin:n.begin,end:n.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:a}),d],illegal:/\[|%/},{begin:/\$[(.]/},e.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+a+"\\()",end:/{/,keywords:"get set",contains:[e.inherit(e.TITLE_MODE,{begin:a}),{begin:/\(\)/},d]}],illegal:/#(?!!)/}}}());hljs.registerLanguage("typescript",function(){"use strict";return function(e){var n={keyword:"in if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const class public private protected get set super static implements enum export import declare type namespace abstract as from extends async await",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document any number boolean string void Promise"},r={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},a={begin:"\\(",end:/\)/,keywords:n,contains:["self",e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.NUMBER_MODE]},t={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,r,a]},s={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:e.C_NUMBER_RE+"n?"}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:n,contains:[]},o={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"xml"}},c={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"css"}},E={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};return i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,c,E,s,e.REGEXP_MODE],{name:"TypeScript",aliases:["ts"],keywords:n,contains:[{className:"meta",begin:/^\s*['"]use strict['"]/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,c,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+e.IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:e.IDENT_RE},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]}]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/[\{;]/,excludeEnd:!0,keywords:n,contains:["self",e.inherit(e.TITLE_MODE,{begin:"[A-Za-z$_][0-9A-Za-z$_]*"}),t],illegal:/%/,relevance:0},{beginKeywords:"constructor",end:/[\{;]/,excludeEnd:!0,contains:["self",t]},{begin:/module\./,keywords:{built_in:"module"},relevance:0},{beginKeywords:"module",end:/\{/,excludeEnd:!0},{beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:"interface extends"},{begin:/\$[(.]/},{begin:"\\."+e.IDENT_RE,relevance:0},r,a]}}}());hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}());hljs.registerLanguage("http",function(){"use strict";return function(e){var n="HTTP/[0-9\\.]+";return{name:"HTTP",aliases:["https"],illegal:"\\S",contains:[{begin:"^"+n,end:"$",contains:[{className:"number",begin:"\\b\\d{3}\\b"}]},{begin:"^[A-Z]+ (.*?) "+n+"$",returnBegin:!0,end:"$",contains:[{className:"string",begin:" ",end:" ",excludeBegin:!0,excludeEnd:!0},{begin:n},{className:"keyword",begin:"[A-Z]+"}]},{className:"attribute",begin:"^\\w",end:": ",excludeEnd:!0,illegal:"\\n|\\s|=",starts:{end:"$",relevance:0}},{begin:"\\n\\n",starts:{subLanguage:[],endsWithParent:!0}}]}}}()); \ No newline at end of file diff --git a/doc/api_assets/hljs.css b/doc/api_assets/hljs.css index de798089557c55..e944148b835ee2 100644 --- a/doc/api_assets/hljs.css +++ b/doc/api_assets/hljs.css @@ -7,6 +7,7 @@ color: #333; } +.hljs-attribute, .hljs-keyword { color: #338; } From f6dff0a57e55dd01b04eee3a74751a0e516144fb Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Tue, 30 Jun 2020 21:44:32 -0700 Subject: [PATCH 035/492] doc: simplify and clarify ReferenceError material in errors.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "should always be considered" -> "indicate" PR-URL: https://github.com/nodejs/node/pull/34151 Reviewed-By: Luigi Pinca Reviewed-By: Anto Aravinth Reviewed-By: Christian Clauss Reviewed-By: Gerhard Stöbich Reviewed-By: Chengzhong Wu Reviewed-By: Andrey Pechkurov --- doc/api/errors.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/api/errors.md b/doc/api/errors.md index d9fd4cd4af2a90..35ba8135aacf50 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -391,8 +391,7 @@ doesNotExist; ``` Unless an application is dynamically generating and running code, -`ReferenceError` instances should always be considered a bug in the code -or its dependencies. +`ReferenceError` instances indicate a bug in the code or its dependencies. ## Class: `SyntaxError` From 1d25e703925f9edb232d44ed21be2b5998ac1b3a Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Tue, 30 Jun 2020 21:48:16 -0700 Subject: [PATCH 036/492] doc: remove "considered" in errors.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "would be considered a `TypeError`" -> "would be a `TypeError`". Using "considered" introduces unnecessary ambiguity. Is is not actually a TypeError but merely "considered" one? Why is that? We don't say. Simplify to "is a TypeError". PR-URL: https://github.com/nodejs/node/pull/34152 Reviewed-By: Luigi Pinca Reviewed-By: Christian Clauss Reviewed-By: Matteo Collina Reviewed-By: Gerhard Stöbich Reviewed-By: Andrey Pechkurov Reviewed-By: Vladimir de Turckheim Reviewed-By: Benjamin Gruenbaum --- doc/api/errors.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/api/errors.md b/doc/api/errors.md index 35ba8135aacf50..af064b089e02ed 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -560,8 +560,7 @@ program. For a comprehensive list, see the [`errno`(3) man page][]. * Extends {errors.Error} Indicates that a provided argument is not an allowable type. For example, -passing a function to a parameter which expects a string would be considered -a `TypeError`. +passing a function to a parameter which expects a string would be a `TypeError`. ```js require('url').parse(() => { }); From bb542176b05cb5efeaf6180345b32bc74a714a4d Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Tue, 30 Jun 2020 09:55:27 -0400 Subject: [PATCH 037/492] test: report actual error code on failure Add a custom message to parallel/test-dgram-error-message-address so that the actual error code that doesn't match the allowed errors is output on assertion failure. PR-URL: https://github.com/nodejs/node/pull/34134 Reviewed-By: Gireesh Punathil Reviewed-By: Anna Henningsen Reviewed-By: Shelley Vohr Reviewed-By: Trivikram Kamat Reviewed-By: Jiawen Geng --- test/parallel/test-dgram-error-message-address.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-dgram-error-message-address.js b/test/parallel/test-dgram-error-message-address.js index 859ee560c34c01..cf243ed2e8560a 100644 --- a/test/parallel/test-dgram-error-message-address.js +++ b/test/parallel/test-dgram-error-message-address.js @@ -47,7 +47,7 @@ socket_ipv6.on('listening', common.mustNotCall()); socket_ipv6.on('error', common.mustCall(function(e) { // EAFNOSUPPORT or EPROTONOSUPPORT means IPv6 is disabled on this system. const allowed = ['EADDRNOTAVAIL', 'EAFNOSUPPORT', 'EPROTONOSUPPORT']; - assert(allowed.includes(e.code)); + assert(allowed.includes(e.code), `'${e.code}' was not one of ${allowed}.`); assert.strictEqual(e.port, undefined); assert.strictEqual(e.message, `bind ${e.code} 111::1`); assert.strictEqual(e.address, '111::1'); From bf4e778e5054f495748174ebba9619202977dea1 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 30 Jun 2020 22:36:10 +0200 Subject: [PATCH 038/492] crypto: move typechecking for timingSafeEqual into C++ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the function more robust against V8 inlining. Fixes: https://github.com/nodejs/node/issues/34073 PR-URL: https://github.com/nodejs/node/pull/34141 Reviewed-By: Richard Lau Reviewed-By: Ujjwal Sharma Reviewed-By: Ben Noordhuis Reviewed-By: Tobias Nießen Reviewed-By: Rich Trott Reviewed-By: Zeyu Yang --- lib/crypto.js | 7 +++-- lib/internal/crypto/util.js | 18 ------------- lib/internal/errors.js | 2 -- src/node_crypto.cc | 26 ++++++++++++++++--- src/node_errors.h | 3 +++ .../test-crypto-timing-safe-equal.js | 4 +-- 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/lib/crypto.js b/lib/crypto.js index 5e45a1cd987260..e4b9d479a50671 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -44,7 +44,11 @@ const { getOptionValue } = require('internal/options'); const pendingDeprecation = getOptionValue('--pending-deprecation'); const { fipsMode } = internalBinding('config'); const fipsForced = getOptionValue('--force-fips'); -const { getFipsCrypto, setFipsCrypto } = internalBinding('crypto'); +const { + getFipsCrypto, + setFipsCrypto, + timingSafeEqual, +} = internalBinding('crypto'); const { randomBytes, randomFill, @@ -101,7 +105,6 @@ const { getHashes, setDefaultEncoding, setEngine, - timingSafeEqual } = require('internal/crypto/util'); const Certificate = require('internal/crypto/certificate'); diff --git a/lib/internal/crypto/util.js b/lib/internal/crypto/util.js index 286efc39a61e4f..8d43514906fde8 100644 --- a/lib/internal/crypto/util.js +++ b/lib/internal/crypto/util.js @@ -9,7 +9,6 @@ const { getCurves: _getCurves, getHashes: _getHashes, setEngine: _setEngine, - timingSafeEqual: _timingSafeEqual } = internalBinding('crypto'); const { @@ -20,7 +19,6 @@ const { hideStackFrames, codes: { ERR_CRYPTO_ENGINE_UNKNOWN, - ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH, ERR_INVALID_ARG_TYPE, } } = require('internal/errors'); @@ -76,21 +74,6 @@ function setEngine(id, flags) { throw new ERR_CRYPTO_ENGINE_UNKNOWN(id); } -function timingSafeEqual(buf1, buf2) { - if (!isArrayBufferView(buf1)) { - throw new ERR_INVALID_ARG_TYPE('buf1', - ['Buffer', 'TypedArray', 'DataView'], buf1); - } - if (!isArrayBufferView(buf2)) { - throw new ERR_INVALID_ARG_TYPE('buf2', - ['Buffer', 'TypedArray', 'DataView'], buf2); - } - if (buf1.byteLength !== buf2.byteLength) { - throw new ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH(); - } - return _timingSafeEqual(buf1, buf2); -} - const getArrayBufferView = hideStackFrames((buffer, name, encoding) => { if (typeof buffer === 'string') { if (encoding === 'buffer') @@ -116,6 +99,5 @@ module.exports = { kHandle, setDefaultEncoding, setEngine, - timingSafeEqual, toBuf }; diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 754ecff1b979c7..1e29735f0db52c 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -800,8 +800,6 @@ E('ERR_CRYPTO_SCRYPT_INVALID_PARAMETER', 'Invalid scrypt parameter', Error); E('ERR_CRYPTO_SCRYPT_NOT_SUPPORTED', 'Scrypt algorithm not supported', Error); // Switch to TypeError. The current implementation does not seem right. E('ERR_CRYPTO_SIGN_KEY_REQUIRED', 'No key provided to sign', Error); -E('ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH', - 'Input buffers must have the same byte length', RangeError); E('ERR_DIR_CLOSED', 'Directory handle was closed', Error); E('ERR_DIR_CONCURRENT_OPERATION', 'Cannot do synchronous work on directory handle with concurrent ' + diff --git a/src/node_crypto.cc b/src/node_crypto.cc index c132e6a089b3cb..4eee56be11034a 100644 --- a/src/node_crypto.cc +++ b/src/node_crypto.cc @@ -6766,10 +6766,30 @@ void StatelessDiffieHellman(const FunctionCallbackInfo& args) { void TimingSafeEqual(const FunctionCallbackInfo& args) { - ArrayBufferViewContents buf1(args[0]); - ArrayBufferViewContents buf2(args[1]); + // Moving the type checking into JS leads to test failures, most likely due + // to V8 inlining certain parts of the wrapper. Therefore, keep them in C++. + // Refs: https://github.com/nodejs/node/issues/34073. + Environment* env = Environment::GetCurrent(args); + if (!args[0]->IsArrayBufferView()) { + THROW_ERR_INVALID_ARG_TYPE( + env, "The \"buf1\" argument must be an instance of " + "Buffer, TypedArray, or DataView."); + return; + } + if (!args[1]->IsArrayBufferView()) { + THROW_ERR_INVALID_ARG_TYPE( + env, "The \"buf2\" argument must be an instance of " + "Buffer, TypedArray, or DataView."); + return; + } + + ArrayBufferViewContents buf1(args[0].As()); + ArrayBufferViewContents buf2(args[1].As()); - CHECK_EQ(buf1.length(), buf2.length()); + if (buf1.length() != buf2.length()) { + THROW_ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH(env); + return; + } return args.GetReturnValue().Set( CRYPTO_memcmp(buf1.data(), buf2.data(), buf1.length()) == 0); diff --git a/src/node_errors.h b/src/node_errors.h index f79b87afd2d525..e23cd164ed719c 100644 --- a/src/node_errors.h +++ b/src/node_errors.h @@ -33,6 +33,7 @@ void OnFatalError(const char* location, const char* message); V(ERR_BUFFER_TOO_LARGE, Error) \ V(ERR_CONSTRUCT_CALL_REQUIRED, TypeError) \ V(ERR_CONSTRUCT_CALL_INVALID, TypeError) \ + V(ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH, RangeError) \ V(ERR_CRYPTO_UNKNOWN_CIPHER, Error) \ V(ERR_CRYPTO_UNKNOWN_DH_GROUP, Error) \ V(ERR_EXECUTION_ENVIRONMENT_NOT_AVAILABLE, Error) \ @@ -86,6 +87,8 @@ void OnFatalError(const char* location, const char* message); "Buffer is not available for the current Context") \ V(ERR_CONSTRUCT_CALL_INVALID, "Constructor cannot be called") \ V(ERR_CONSTRUCT_CALL_REQUIRED, "Cannot call constructor without `new`") \ + V(ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH, \ + "Input buffers must have the same byte length") \ V(ERR_CRYPTO_UNKNOWN_CIPHER, "Unknown cipher") \ V(ERR_CRYPTO_UNKNOWN_DH_GROUP, "Unknown DH group") \ V(ERR_EXECUTION_ENVIRONMENT_NOT_AVAILABLE, \ diff --git a/test/sequential/test-crypto-timing-safe-equal.js b/test/sequential/test-crypto-timing-safe-equal.js index f9709ac966d6a1..769b5f7d8810c3 100644 --- a/test/sequential/test-crypto-timing-safe-equal.js +++ b/test/sequential/test-crypto-timing-safe-equal.js @@ -48,7 +48,7 @@ assert.throws( name: 'TypeError', message: 'The "buf1" argument must be an instance of Buffer, TypedArray, or ' + - "DataView. Received type string ('not a buffer')" + 'DataView.' } ); @@ -59,6 +59,6 @@ assert.throws( name: 'TypeError', message: 'The "buf2" argument must be an instance of Buffer, TypedArray, or ' + - "DataView. Received type string ('not a buffer')" + 'DataView.' } ); From 44246e67010f435648818975c5f1609eb14fc059 Mon Sep 17 00:00:00 2001 From: Xu Meng Date: Mon, 29 Jun 2020 05:54:23 -0500 Subject: [PATCH 039/492] test: skip some IBM i unsupported test cases Issuing a shutdown() on IBM i PASE with parameter SHUT_WR also sends a normal close sequence to the partner program. This leads to timing issues and ECONNRESET failures in some test cases. Refs: https://github.com/libuv/libuv/pull/2782 PR-URL: https://github.com/nodejs/node/pull/34118 Reviewed-By: Richard Lau Reviewed-By: Beth Griggs Reviewed-By: James M Snell --- test/parallel/parallel.status | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/parallel/parallel.status b/test/parallel/parallel.status index b72cd30775c05c..94d15064b38423 100644 --- a/test/parallel/parallel.status +++ b/test/parallel/parallel.status @@ -65,3 +65,13 @@ test-cluster-shared-leak: SKIP test-http-writable-true-after-close: SKIP test-http2-connect-method: SKIP test-net-error-twice: SKIP +# https://github.com/libuv/libuv/pull/2782 +test-net-allow-half-open: SKIP +test-net-keepalive: SKIP +test-net-persistent-keepalive: SKIP +test-net-socket-close-after-end: SKIP +test-net-socket-connect-without-cb: SKIP +test-net-socket-connecting: SKIP +test-net-socket-ready-without-cb: SKIP +test-net-write-after-end-nt: SKIP +test-tls-env-extra-ca: SKIP \ No newline at end of file From a9302b50c9be969112cac4128902e8860683e527 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Wed, 1 Jul 2020 06:21:48 -0700 Subject: [PATCH 040/492] doc: simply dns.ADDRCONFIG language PR-URL: https://github.com/nodejs/node/pull/34155 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell --- doc/api/dns.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/api/dns.md b/doc/api/dns.md index 021e765ba46d77..6b078a58997d6e 100644 --- a/doc/api/dns.md +++ b/doc/api/dns.md @@ -225,10 +225,9 @@ changes: The following flags can be passed as hints to [`dns.lookup()`][]. -* `dns.ADDRCONFIG`: Returned address types are determined by the types -of addresses supported by the current system. For example, IPv4 addresses -are only returned if the current system has at least one IPv4 address -configured. Loopback addresses are not considered. +* `dns.ADDRCONFIG`: Limits returned address types to the types of non-loopback +addresses configured on the system. For example, IPv4 addresses are only +returned if the current system has at least one IPv4 address configured. * `dns.V4MAPPED`: If the IPv6 family was specified, but no IPv6 addresses were found, then return IPv4 mapped IPv6 addresses. It is not supported on some operating systems (e.g FreeBSD 10.1). From 49383c8a252aa997414521f8e1d16b55dd06db80 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Wed, 1 Jul 2020 19:55:46 -0700 Subject: [PATCH 041/492] doc: improve triaging text in issues.md Remove a double negative and many superfluous sentences. The sentiment in the removed material is good, but we already tell people that discussion should be focused, helpful, and professional. Having two extra paragraphs greatly reduces the likelihood that people will read the material. PR-URL: https://github.com/nodejs/node/pull/34164 Reviewed-By: Anto Aravinth Reviewed-By: James M Snell --- doc/guides/contributing/issues.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/doc/guides/contributing/issues.md b/doc/guides/contributing/issues.md index a021e4c8207c7d..5e123f03e10117 100644 --- a/doc/guides/contributing/issues.md +++ b/doc/guides/contributing/issues.md @@ -84,23 +84,11 @@ See [How to create a Minimal, Complete, and Verifiable example](https://stackove ## Triaging a Bug Report -Once an issue has been opened, it is not uncommon for there to be discussion +Once an issue has been opened, it is common for there to be discussion around it. Some contributors may have differing opinions about the issue, including whether the behavior being seen is a bug or a feature. This discussion is part of the process and should be kept focused, helpful, and professional. -Short, clipped responses that provide neither additional context nor supporting -details are not helpful or professional to many. Such responses are simply -annoying and unfriendly. - -Contributors are encouraged to help one another to make forward progress as much -as possible, empowering one another to solve issues collaboratively. If you -choose to comment on an issue that you feel either is not a problem that needs -to be fixed, or if you encounter information in an issue that you feel is -incorrect, explain *why* you feel that way with additional supporting context, -and be willing to be convinced that you may be wrong. By doing so, we can often -reach the correct outcome much faster. - ## Resolving a Bug Report In the vast majority of cases, issues are resolved by opening a Pull Request. From 8fb265d03c2c3ba4261daa2ac19445d9ad68591d Mon Sep 17 00:00:00 2001 From: Derek Lewis Date: Sun, 14 Jun 2020 19:30:38 -0400 Subject: [PATCH 042/492] doc: clarify esm conditional exports prose This commit clarifies the behavior of a couple aspects of conditional exports that may have been difficult to grasp from the prose alone. PR-URL: https://github.com/nodejs/node/pull/33886 Reviewed-By: Guy Bedford Reviewed-By: Benjamin Gruenbaum Reviewed-By: Anna Henningsen Reviewed-By: James M Snell --- doc/api/esm.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index c8404a9f6892e6..0691a5cce3d5da 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -419,7 +419,7 @@ For example, a package that wants to provide different ES module exports for } ``` -Node.js supports the following conditions: +Node.js supports the following conditions out of the box: * `"import"` - matched when the package is loaded via `import` or `import()`. Can reference either an ES module or CommonJS file, as both @@ -434,18 +434,18 @@ Node.js supports the following conditions: * `"default"` - the generic fallback that will always match. Can be a CommonJS or ES module file. _This condition should always come last._ -Condition matching is applied in object order from first to last within the -`"exports"` object. _The general rule is that conditions should be used -from most specific to least specific in object order._ +Within the `"exports"` object, key order is significant. During condition +matching, earlier entries have higher priority and take precedence over later +entries. _The general rule is that conditions should be from most specific to +least specific in object order_. Other conditions such as `"browser"`, `"electron"`, `"deno"`, `"react-native"`, -etc. are ignored by Node.js but may be used by other runtimes or tools. -Further restrictions, definitions or guidance on condition names may be -provided in the future. +etc. are unknown to, and thus ignored by Node.js. Runtimes or tools other than +Node.js may use them at their discretion. Further restrictions, definitions, or +guidance on condition names may occur in the future. Using the `"import"` and `"require"` conditions can lead to some hazards, -which are explained further in -[the dual CommonJS/ES module packages section][]. +which are further explained in [the dual CommonJS/ES module packages section][]. Conditional exports can also be extended to exports subpaths, for example: @@ -1128,10 +1128,11 @@ The `conditions` property on the `context` is an array of conditions for for looking up conditional mappings elsewhere or to modify the list when calling the default resolution logic. -The [current set of Node.js default conditions][Conditional exports] will always -be in the `context.conditions` list passed to the hook. If the hook wants to -ensure Node.js-compatible resolution logic, all items from this default -condition list **must** be passed through to the `defaultResolve` function. +The current [package exports conditions][Conditional Exports] will always be in +the `context.conditions` array passed into the hook. To guarantee _default +Node.js module specifier resolution behavior_ when calling `defaultResolve`, the +`context.conditions` array passed to it _must_ include _all_ elements of the +`context.conditions` array originally passed into the `resolve` hook. ```js /** From f12c6f406a83a0802101a138f12fb0fdc320ae86 Mon Sep 17 00:00:00 2001 From: Denys Otrishko Date: Thu, 4 Jun 2020 20:58:52 +0300 Subject: [PATCH 043/492] doc: improve async_hooks asynchronous context example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use writeFile(1) everywhere to log * prettify execution id graph * add clearer explanation for TickObject presence * add causation graph via triggerAsyncId PR-URL: https://github.com/nodejs/node/pull/33730 Reviewed-By: Gerhard Stöbich Reviewed-By: Andrey Pechkurov Reviewed-By: James M Snell Reviewed-By: Vladimir de Turckheim --- doc/api/async_hooks.md | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/doc/api/async_hooks.md b/doc/api/async_hooks.md index de83dacaf46738..8a9835626b4095 100644 --- a/doc/api/async_hooks.md +++ b/doc/api/async_hooks.md @@ -336,20 +336,17 @@ async_hooks.createHook({ }, before(asyncId) { const indentStr = ' '.repeat(indent); - fs.writeFileSync('log.out', - `${indentStr}before: ${asyncId}\n`, { flag: 'a' }); + fs.writeSync(process.stdout.fd, `${indentStr}before: ${asyncId}\n`); indent += 2; }, after(asyncId) { indent -= 2; const indentStr = ' '.repeat(indent); - fs.writeFileSync('log.out', - `${indentStr}after: ${asyncId}\n`, { flag: 'a' }); + fs.writeSync(process.stdout.fd, `${indentStr}after: ${asyncId}\n`); }, destroy(asyncId) { const indentStr = ' '.repeat(indent); - fs.writeFileSync('log.out', - `${indentStr}destroy: ${asyncId}\n`, { flag: 'a' }); + fs.writeSync(process.stdout.fd, `${indentStr}destroy: ${asyncId}\n`); }, }).enable(); @@ -385,16 +382,38 @@ the value of the current execution context; which is delineated by calls to Only using `execution` to graph resource allocation results in the following: ```console -Timeout(7) -> TickObject(6) -> root(1) + root(1) + ^ + | +TickObject(6) + ^ + | + Timeout(7) ``` The `TCPSERVERWRAP` is not part of this graph, even though it was the reason for `console.log()` being called. This is because binding to a port without a host name is a *synchronous* operation, but to maintain a completely asynchronous -API the user's callback is placed in a `process.nextTick()`. +API the user's callback is placed in a `process.nextTick()`. Which is why +`TickObject` is present in the output and is a 'parent' for `.listen()` +callback. The graph only shows *when* a resource was created, not *why*, so to track -the *why* use `triggerAsyncId`. +the *why* use `triggerAsyncId`. Which can be represented with the following +graph: + +```console + bootstrap(1) + | + ˅ +TCPSERVERWRAP(5) + | + ˅ + TickObject(6) + | + ˅ + Timeout(7) +``` ##### `before(asyncId)` From 5f131f71e9021ddee14a757df211280f205c10ce Mon Sep 17 00:00:00 2001 From: Rodion Abdurakhimov Date: Sun, 31 May 2020 01:09:58 +0300 Subject: [PATCH 044/492] doc: fix source link margin to sub-header mark Prior to this commit, [src] link overlapped sharped (#) sub-header link PR-URL: https://github.com/nodejs/node/pull/33664 Reviewed-By: James M Snell --- doc/api_assets/style.css | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api_assets/style.css b/doc/api_assets/style.css index 37e7376d0583f2..a143fd88013b4c 100644 --- a/doc/api_assets/style.css +++ b/doc/api_assets/style.css @@ -342,6 +342,7 @@ h5 { .srclink { float: right; font-size: smaller; + margin-right: 30px; } h1 span, From 25939ccded57a4b49d53ef734249a36ea9892f03 Mon Sep 17 00:00:00 2001 From: Gabriel Schulhof Date: Mon, 29 Jun 2020 15:10:31 -0700 Subject: [PATCH 045/492] doc: fix entry for `napi_create_external_buffer` Remove text regarding copying, because `napi_create_external_buffer` does not copy. Fixes: https://github.com/nodejs/node/issues/33471 PR-URL: https://github.com/nodejs/node/pull/34125 Reviewed-By: Anna Henningsen Reviewed-By: Mathias Buus Reviewed-By: Zeyu Yang Reviewed-By: Chengzhong Wu Reviewed-By: Michael Dawson Reviewed-By: James M Snell --- doc/api/n-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/n-api.md b/doc/api/n-api.md index 7002dfef61baa3..cfd4777c713c85 100644 --- a/doc/api/n-api.md +++ b/doc/api/n-api.md @@ -2038,7 +2038,7 @@ napi_status napi_create_external_buffer(napi_env env, * `[in] env`: The environment that the API is invoked under. * `[in] length`: Size in bytes of the input buffer (should be the same as the size of the new buffer). -* `[in] data`: Raw pointer to the underlying buffer to copy from. +* `[in] data`: Raw pointer to the underlying buffer to expose to JavaScript. * `[in] finalize_cb`: Optional callback to call when the `ArrayBuffer` is being collected. [`napi_finalize`][] provides more details. * `[in] finalize_hint`: Optional hint to pass to the finalize callback during From 71664158fca12fadd11c3f6462341fedbed8f4ef Mon Sep 17 00:00:00 2001 From: Anentropic Date: Mon, 22 Apr 2019 20:37:48 +0100 Subject: [PATCH 046/492] doc: clarify how to read process.stdin document more clearly that stdin will emit multiple readable events PR-URL: https://github.com/nodejs/node/pull/27350 Reviewed-By: Matteo Collina Reviewed-By: Robert Nagy Reviewed-By: James M Snell --- doc/api/process.md | 17 ++--------------- doc/api/stream.md | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/doc/api/process.md b/doc/api/process.md index 3835401c2e5fdb..9271c92623a79c 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -2226,21 +2226,7 @@ The `process.stdin` property returns a stream connected to stream) unless fd `0` refers to a file, in which case it is a [Readable][] stream. -```js -process.stdin.setEncoding('utf8'); - -process.stdin.on('readable', () => { - let chunk; - // Use a loop to make sure we read all available data. - while ((chunk = process.stdin.read()) !== null) { - process.stdout.write(`data: ${chunk}`); - } -}); - -process.stdin.on('end', () => { - process.stdout.write('end'); -}); -``` +For details of how to read from `stdin` see [`readable.read()`][]. As a [Duplex][] stream, `process.stdin` can also be used in "old" mode that is compatible with scripts written for Node.js prior to v0.10. @@ -2594,6 +2580,7 @@ cases: [Event Loop]: https://nodejs.org/en/docs/guides/event-loop-timers-and-nexttick/#process-nexttick [LTS]: https://github.com/nodejs/Release [Readable]: stream.html#stream_readable_streams +[`readable.read()`]: stream.html#stream_readable_read_size [Signal Events]: #process_signal_events [Stream compatibility]: stream.html#stream_compatibility_with_older_node_js_versions [TTY]: tty.html#tty_tty diff --git a/doc/api/stream.md b/doc/api/stream.md index d4f7029dd8b422..92871539645459 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -1108,17 +1108,48 @@ automatically until the internal buffer is fully drained. ```js const readable = getReadableStreamSomehow(); + +// 'readable' may be triggered multiple times as data is buffered in readable.on('readable', () => { let chunk; + console.log('Stream is readable (new data received in buffer)'); + // Use a loop to make sure we read all currently available data while (null !== (chunk = readable.read())) { - console.log(`Received ${chunk.length} bytes of data.`); + console.log(`Read ${chunk.length} bytes of data...`); } }); + +// 'end' will be triggered once when there is no more data available +readable.on('end', () => { + console.log('Reached end of stream.'); +}); ``` -The `while` loop is necessary when processing data with -`readable.read()`. Only after `readable.read()` returns `null`, -[`'readable'`][] will be emitted. +Each call to `readable.read()` returns a chunk of data, or `null`. The chunks +are not concatenated. A `while` loop is necessary to consume all data +currently in the buffer. When reading a large file `.read()` may return `null`, +having consumed all buffered content so far, but there is still more data to +come not yet buffered. In this case a new `'readable'` event will be emitted +when there is more data in the buffer. Finally the `'end'` event will be +emitted when there is no more data to come. + +Therefore to read a file's whole contents from a `readable`, it is necessary +to collect chunks across multiple `'readable'` events: + +```js +const chunks = []; + +readable.on('readable', () => { + let chunk; + while (null !== (chunk = readable.read())) { + chunks.push(chunk); + } +}); + +readable.on('end', () => { + const content = chunks.join(''); +}); +``` A `Readable` stream in object mode will always return a single item from a call to [`readable.read(size)`][stream-read], regardless of the value of the From 75637e686740478c198889b02315c76543ed8335 Mon Sep 17 00:00:00 2001 From: Saleem Date: Sat, 16 Nov 2019 18:13:18 -0500 Subject: [PATCH 047/492] doc: use consistent naming in stream doc Consistency of method naming referred to as readable.push several other times in transform documentation and also documented under readable, so makes sense to just stick with readable.push PR-URL: https://github.com/nodejs/node/pull/30506 Reviewed-By: Matteo Collina Reviewed-By: James M Snell --- doc/api/stream.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/stream.md b/doc/api/stream.md index 92871539645459..d7c8ebc266bd86 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -2554,7 +2554,7 @@ method. This will be called when there is no more written data to be consumed, but before the [`'end'`][] event is emitted signaling the end of the [`Readable`][] stream. -Within the `transform._flush()` implementation, the `readable.push()` method +Within the `transform._flush()` implementation, the `transform.push()` method may be called zero or more times, as appropriate. The `callback` function must be called when the flush operation is complete. @@ -2583,7 +2583,7 @@ methods only. All `Transform` stream implementations must provide a `_transform()` method to accept input and produce output. The `transform._transform()` implementation handles the bytes being written, computes an output, then passes -that output off to the readable portion using the `readable.push()` method. +that output off to the readable portion using the `transform.push()` method. The `transform.push()` method may be called zero or more times to generate output from a single input chunk, depending on how much is to be output @@ -2595,7 +2595,7 @@ The `callback` function must be called only when the current chunk is completely consumed. The first argument passed to the `callback` must be an `Error` object if an error occurred while processing the input or `null` otherwise. If a second argument is passed to the `callback`, it will be forwarded on to the -`readable.push()` method. In other words, the following are equivalent: +`transform.push()` method. In other words, the following are equivalent: ```js transform.prototype._transform = function(data, encoding, callback) { From 4ac0df9160a837b0fbe61135aa7556865270a04d Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Fri, 29 May 2020 09:43:20 -0700 Subject: [PATCH 048/492] doc: no longer maintain a CNA structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node.js hasn't touched the cve-management repo since the Feb 2019 security release, we've used the HackerOne CVE allocation process. Maintaining our status as a CNA is not zero cost, there is some routine adminstration that is requested (see this doc for details). As we no longer use the CVE management process, I propose removing it. If this lands, I will go through the interactions with Mitre so that Node.js is no longer a CNA and cleanup related resources (email aliases, archive the cve-management repo, whatever else I find). PR-URL: https://github.com/nodejs/node/pull/33639 Reviewed-By: James M Snell Reviewed-By: Vladimir de Turckheim Reviewed-By: Ruben Bridgewater Reviewed-By: Matteo Collina Reviewed-By: Beth Griggs Reviewed-By: Michael Dawson Reviewed-By: Сковорода Никита Андреевич --- doc/guides/cve-management-process.md | 141 --------------------------- 1 file changed, 141 deletions(-) delete mode 100644 doc/guides/cve-management-process.md diff --git a/doc/guides/cve-management-process.md b/doc/guides/cve-management-process.md deleted file mode 100644 index fbd7f2cb84a3be..00000000000000 --- a/doc/guides/cve-management-process.md +++ /dev/null @@ -1,141 +0,0 @@ -# Node.js CVE management process - -The Node.js project acts as a [Common Vulnerabilities and Exposures (CVE) -Numbering Authority (CNA)](https://cve.mitre.org/cve/cna.html). -The current scope is for all actively developed versions of software -developed under the Node.js project (ie. ). -This means that the Node.js team reviews CVE requests and if appropriate -assigns CVE numbers to vulnerabilities. The scope currently **does not** -include third party modules. - -More detailed information about the CNA program is available in -[CNA_Rules_v1.1](https://cve.mitre.org/cve/cna/CNA_Rules_v1.1.pdf). - -## Contacts - -As part of the CNA program the Node.js team must provide a number -of contact points. Email aliases have been setup for these as follows: - -* **Public contact points**. Email address to which people will be directed - by Mitre when they are asked for a way to contact the Node.js team about - CVE-related issues. **[cve-request@iojs.org][]** - -* **Private contact points**. Administrative contacts that Mitre can reach out - to directly in case there are issues that require immediate attention. - **[cve-mitre-contact@iojs.org][]** - -* **Email addresses to add to the CNA email discussion list**. This address has - been added to a closed mailing list that is used for announcements, - sharing documents, or discussion relevant to the CNA community. - The list rarely has more than ten messages a week. - **[cna-discussion-list@iojs.org][]** - -## CNA management processes - -### CVE Block management - -The CNA program allows the Node.js team to request a block of CVEs in -advance. These CVEs are managed in a repository within the Node.js -private organization called -[cve-management](https://github.com/nodejs-private/cve-management). -For each year there will be a markdown file titled "cve-management-XXXX" -where XXXX is the year (for example 'cve-management-2017.md'). - -This file will have the following sections: - -* Available -* Pending -* Announced - -When a new block of CVEs is received from Mitre they will be listed under -the `Available` section. - -These CVEs will be moved from the Available to Pending and Announced -as outlined in the section titled `CVE Management process`. - -In addition, when moving a CVE from Available such that there are less -than two remaining CVEs a new block must be requested as follows: - -* Use the Mitre request form with the - option `Request a Block of IDs` to request a new block. -* The new block will be sent to the requester through email. -* Once the new block has been received, the requester will add them - to the Available list. - -All changes to the files for managing CVEs in a given year will -be done through pull requests so that we have a record of how -the CVEs have been assigned. - -CVEs are only valid for a specific year. At the beginning of each -year the old CVEs should be removed from the list. A new block -of CVEs should then be requested using the steps listed above. - -### External CVE request process - -When a request for a CVE is received via the [cve-request@iojs.org][] -email alias the following process will be followed (likely updated -after we get HackerOne up and running). - -* Respond to the requester indicating that we have the request - and will review soon. -* Open an issue in the security repo for the request. -* Review the request. - * If a CVE is appropriate then assign the - CVE as outline in the section titled - [CVE Management process for Node.js vulnerabilities](#cve-management-process-for-nodejs-vulnerabilities) - and return the CVE number to the requester (along with the request - to keep it confidential until the vulnerability is announced) - * If a CVE is not appropriate then respond to the requester - with the details as to why. - -### Quarterly reporting - -* There is a requirement for quarterly reports to Mitre on CVE - activity. Not sure of the specific requirements yet. Will - add details on process once we've done the first one. - -## CVE Management process for Node.js vulnerabilities - -When the Node.js team is going announce a new vulnerability the -following steps are used to assign, announce and report a CVE. - -* Select the next CVE in the block available from the CNA process as - outlined in the section above. -* Move the CVE from the unassigned block, to the Pending section along - with a link to the issue in the security repo that is being used - to discuss the vulnerability. -* As part of the - [security announcement process](https://github.com/nodejs/security-wg/blob/master/processes/security_annoucement_process.md), - in the security issue being used to discuss the - vulnerability, associate the CVE to that vulnerability. This is most - commonly done by including it in the draft for the announcement that - will go out once the associated security releases are available. -* Once the security announcement goes out: - * Use the [Mitre form](https://cveform.mitre.org/) to report the - CVE details to Mitre using the `Notify CVE about a publication`. The - link to the advisory will be the for the blog announcing that security - releases are available. The description should be a subset of the - details in that blog. - - Ensure that the contact address entered in the form is - `cve-mitre-contact@iojs.org`. Anything else may require slow, manual - verification of the identity of the CVE submitter. - - For each CVE listed, the additional data must include the following fields - updated with appropriate data for the CVE: - ```text - [CVEID]: CVE-XXXX-XXXX - [PRODUCT]: Node.js - [VERSION]: 8.x+, 9.x+, 10.x+ - [PROBLEMTYPE]: Denial of Service - [REFERENCES]: Link to the blog for the final announce - [DESCRIPTION]: Description from final announce - [ASSIGNINGCNA]: Node.js Foundation - ``` -* Move the CVE from the Pending section to the Announced section along - with a link to the Node.js blog post announcing that releases - are available. - -[cve-request@iojs.org]: mailto:cve-request@iojs.org -[cve-mitre-contact@iojs.org]: mailto:cve-mitre-contact@iojs.org -[cna-discussion-list@iojs.org]: mailto:cna-discussion-list@iojs.org From 8960a63312c684aa9c840a9f9c06fecbfa04427e Mon Sep 17 00:00:00 2001 From: Mateusz Krawczuk Date: Thu, 25 Jun 2020 15:50:17 +0200 Subject: [PATCH 049/492] doc: add a reference to the list of OpenSSL flags. Some of the SSL_OP_* constants are missing description in the documentation. Instead of rewriting the description from OpenSSL's wiki, I have decided to put a link to a detailed list in the 'OpenSSL Options' section. I see no point of doing both - adding a reference to the wiki and adding constant descriptions - but I might do if presented with convincing arguments. This is a follow-up to #33929. PR-URL: https://github.com/nodejs/node/pull/34050 Reviewed-By: James M Snell --- doc/api/crypto.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/api/crypto.md b/doc/api/crypto.md index e087c12479dc25..b804b9bb4be909 100644 --- a/doc/api/crypto.md +++ b/doc/api/crypto.md @@ -3358,6 +3358,8 @@ the `crypto`, `tls`, and `https` modules and are generally specific to OpenSSL. ### Other OpenSSL constants +See the [list of SSL OP Flags][] for details. + @@ -3525,3 +3527,4 @@ the `crypto`, `tls`, and `https` modules and are generally specific to OpenSSL. [scrypt]: https://en.wikipedia.org/wiki/Scrypt [stream-writable-write]: stream.html#stream_writable_write_chunk_encoding_callback [stream]: stream.html +[list of SSL OP Flags]: wiki.openssl.org/index.php/List_of_SSL_OP_Flags#Table_of_Options From 32ef1b3347aa0b596cbefe7684badded0b026e92 Mon Sep 17 00:00:00 2001 From: Luigi Pinca Date: Sun, 28 Jun 2020 09:50:10 +0200 Subject: [PATCH 050/492] doc: clarify that the ctx argument is optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify that the `ctx` argument of the `SNICallback` callback is optional. Fixes: https://github.com/nodejs/node/issues/34085 PR-URL: https://github.com/nodejs/node/pull/34097 Reviewed-By: Ujjwal Sharma Reviewed-By: Tobias Nießen Reviewed-By: Trivikram Kamat Reviewed-By: James M Snell --- doc/api/tls.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/api/tls.md b/doc/api/tls.md index 65b35cffe96101..c652805d89c1ab 100644 --- a/doc/api/tls.md +++ b/doc/api/tls.md @@ -1687,12 +1687,15 @@ changes: * `sessionTimeout` {number} The number of seconds after which a TLS session created by the server will no longer be resumable. See [Session Resumption][] for more information. **Default:** `300`. - * `SNICallback(servername, cb)` {Function} A function that will be called if - the client supports SNI TLS extension. Two arguments will be passed when - called: `servername` and `cb`. `SNICallback` should invoke `cb(null, ctx)`, - where `ctx` is a `SecureContext` instance. (`tls.createSecureContext(...)` - can be used to get a proper `SecureContext`.) If `SNICallback` wasn't - provided the default callback with high-level API will be used (see below). + * `SNICallback(servername, callback)` {Function} A function that will be + called if the client supports SNI TLS extension. Two arguments will be + passed when called: `servername` and `callback`. `callback` is an + error-first callback that takes two optional arguments: `error` and `ctx`. + `ctx`, if provided, is a `SecureContext` instance. + [`tls.createSecureContext()`][] can be used to get a proper `SecureContext`. + If `callback` is called with a falsy `ctx` argument, the default secure + context of the server will be used. If `SNICallback` wasn't provided the + default callback with high-level API will be used (see below). * `ticketKeys`: {Buffer} 48-bytes of cryptographically strong pseudo-random data. See [Session Resumption][] for more information. * `pskCallback` {Function} From 089a4479a45af21264f2990e7ac94b5faf6de5dd Mon Sep 17 00:00:00 2001 From: Julien Poissonnier Date: Mon, 29 Jun 2020 15:46:04 +0200 Subject: [PATCH 051/492] doc: update wording in "Two reading modes" PR-URL: https://github.com/nodejs/node/pull/34119 Reviewed-By: Gireesh Punathil Reviewed-By: Zeyu Yang Reviewed-By: Luigi Pinca Reviewed-By: James M Snell --- doc/api/stream.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/api/stream.md b/doc/api/stream.md index d7c8ebc266bd86..6f400b51eba058 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -700,8 +700,8 @@ instance, when the `readable.resume()` method is called without a listener attached to the `'data'` event, or when a `'data'` event handler is removed from the stream. -Adding a [`'readable'`][] event handler automatically make the stream to -stop flowing, and the data to be consumed via +Adding a [`'readable'`][] event handler automatically makes the stream +stop flowing, and the data has to be consumed via [`readable.read()`][stream-read]. If the [`'readable'`][] event handler is removed, then the stream will start flowing again if there is a [`'data'`][] event handler. From 050753527752b9018c15dbcba1837aa871d62f10 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Fri, 13 Dec 2019 18:38:02 -0500 Subject: [PATCH 052/492] tools: remove lint-js.js lint-js.js was implemented before ESLint had a caching feature. It is now only used in CI. Let's remove it on the following grounds: * It results in occasional (and puzzling) yellow CI runs for node-test-linter because the tap file is corrupted somehow. Interleaved maybe? I don't know, but a simple solution is removing it and running ESLint directly. * On my local laptop, it reduces the linting from about 75 seconds to about 55 seconds. This kind of savings is not worth the added complexity and the instability noted above. PR-URL: https://github.com/nodejs/node/pull/30955 Reviewed-By: Richard Lau Reviewed-By: Jiawen Geng Reviewed-By: Daijiro Wachi Reviewed-By: Anatoli Papirovski Reviewed-By: Michael Dawson Reviewed-By: James M Snell --- Makefile | 5 +- tools/lint-js.js | 272 ----------------------------------------------- vcbuild.bat | 10 +- 3 files changed, 4 insertions(+), 283 deletions(-) delete mode 100644 tools/lint-js.js diff --git a/Makefile b/Makefile index 32a8a6c2dca732..a07e0e630287a8 100644 --- a/Makefile +++ b/Makefile @@ -1239,8 +1239,9 @@ lint-js: jslint: lint-js @echo "Please use lint-js instead of jslint" -run-lint-js-ci = tools/lint-js.js $(PARALLEL_ARGS) -f tap -o test-eslint.tap \ - $(LINT_JS_TARGETS) +run-lint-js-ci = tools/node_modules/eslint/bin/eslint.js \ + --report-unused-disable-directives --ext=.js,.mjs,.md -f tap \ + -o test-eslint.tap $(LINT_JS_TARGETS) .PHONY: lint-js-ci # On the CI the output is emitted in the TAP format. diff --git a/tools/lint-js.js b/tools/lint-js.js deleted file mode 100644 index 4ddbe228d16d37..00000000000000 --- a/tools/lint-js.js +++ /dev/null @@ -1,272 +0,0 @@ -'use strict'; - -const rulesDirs = ['tools/eslint-rules']; -const extensions = ['.js', '.mjs', '.md']; -// This is the maximum number of files to be linted per worker at any given time -const maxWorkload = 60; - -const cluster = require('cluster'); -const path = require('path'); -const fs = require('fs'); -const totalCPUs = require('os').cpus().length; - -const CLIEngine = require('eslint').CLIEngine; -const glob = require('eslint/node_modules/glob'); - -const cliOptions = { - rulePaths: rulesDirs, - extensions: extensions, -}; - -// Check if we should fix errors that are fixable -if (process.argv.indexOf('-F') !== -1) - cliOptions.fix = true; - -const cli = new CLIEngine(cliOptions); - -if (cluster.isMaster) { - let numCPUs = 1; - const paths = []; - let files = null; - let totalPaths = 0; - let failures = 0; - let successes = 0; - let lastLineLen = 0; - let curPath = 'Starting ...'; - let showProgress = true; - const globOptions = { - nodir: true, - ignore: '**/node_modules/**/*' - }; - const workerConfig = {}; - let startTime; - let formatter; - let outFn; - let fd; - let i; - - // Check if spreading work among all cores/cpus - if (process.argv.indexOf('-J') !== -1) - numCPUs = totalCPUs; - - // Check if spreading work among an explicit number of cores/cpus - i = process.argv.indexOf('-j'); - if (i !== -1) { - if (!process.argv[i + 1]) - throw new Error('Missing parallel job count'); - numCPUs = parseInt(process.argv[i + 1], 10); - if (!isFinite(numCPUs) || numCPUs <= 0) - throw new Error('Bad parallel job count'); - } - - // Check for custom ESLint report formatter - i = process.argv.indexOf('-f'); - if (i !== -1) { - if (!process.argv[i + 1]) - throw new Error('Missing format name'); - const format = process.argv[i + 1]; - formatter = cli.getFormatter(format); - if (!formatter) - throw new Error('Invalid format name'); - // Automatically disable progress display - showProgress = false; - // Tell worker to send all results, not just linter errors - workerConfig.sendAll = true; - } else { - // Use default formatter - formatter = cli.getFormatter(); - } - - // Check if outputting ESLint report to a file instead of stdout - i = process.argv.indexOf('-o'); - if (i !== -1) { - if (!process.argv[i + 1]) - throw new Error('Missing output filename'); - const outPath = path.resolve(process.argv[i + 1]); - fd = fs.openSync(outPath, 'w'); - outFn = function(str) { - fs.writeSync(fd, str, 'utf8'); - }; - process.on('exit', () => { fs.closeSync(fd); }); - } else { - outFn = function(str) { - process.stdout.write(str); - }; - } - - // Process the rest of the arguments as paths to lint, ignoring any unknown - // flags - for (i = 2; i < process.argv.length; ++i) { - if (process.argv[i][0] === '-') { - switch (process.argv[i]) { - case '-f': // Skip format name - case '-o': // Skip filename - case '-j': // Skip parallel job count number - ++i; - break; - } - continue; - } - paths.push(process.argv[i]); - } - - if (paths.length === 0) - return; - totalPaths = paths.length; - - if (showProgress) { - // Start the progress display update timer when the first worker is ready - cluster.once('online', () => { - startTime = process.hrtime(); - setInterval(printProgress, 1000).unref(); - printProgress(); - }); - } - - cluster.on('online', (worker) => { - // Configure worker and give it some initial work to do - worker.send(workerConfig); - sendWork(worker); - }); - - process.on('exit', (code) => { - if (showProgress) { - curPath = 'Done'; - printProgress(); - outFn('\r\n'); - } - if (code === 0) - process.exit(failures ? 1 : 0); - }); - - for (i = 0; i < numCPUs; ++i) - cluster.fork().on('message', onWorkerMessage).on('exit', onWorkerExit); - - function onWorkerMessage(results) { - if (typeof results !== 'number') { - // The worker sent us results that are not all successes - if (workerConfig.sendAll) { - failures += results.errorCount; - results = results.results; - } else { - failures += results.length; - } - outFn(`${formatter(results)}\r\n`); - printProgress(); - } else { - successes += results; - } - // Try to give the worker more work to do - sendWork(this); - } - - function onWorkerExit(code, signal) { - if (code !== 0 || signal) - process.exit(2); - } - - function sendWork(worker) { - if (!files || !files.length) { - // We either just started or we have no more files to lint for the current - // path. Find the next path that has some files to be linted. - while (paths.length) { - let dir = paths.shift(); - curPath = dir; - const patterns = cli.resolveFileGlobPatterns([dir]); - dir = path.resolve(patterns[0]); - files = glob.sync(dir, globOptions); - if (files.length) - break; - } - if ((!files || !files.length) && !paths.length) { - // We exhausted all input paths and thus have nothing left to do, so end - // the worker - return worker.disconnect(); - } - } - // Give the worker an equal portion of the work left for the current path, - // but not exceeding a maximum file count in order to help keep *all* - // workers busy most of the time instead of only a minority doing most of - // the work. - const sliceLen = Math.min(maxWorkload, Math.ceil(files.length / numCPUs)); - let slice; - if (sliceLen === files.length) { - // Micro-optimization to avoid splicing to an empty array - slice = files; - files = null; - } else { - slice = files.splice(0, sliceLen); - } - worker.send(slice); - } - - function printProgress() { - if (!showProgress) - return; - - // Clear line - outFn(`\r ${' '.repeat(lastLineLen)}\r`); - - // Calculate and format the data for displaying - const elapsed = process.hrtime(startTime)[0]; - const mins = `${Math.floor(elapsed / 60)}`.padStart(2, '0'); - const secs = `${elapsed % 60}`.padStart(2, '0'); - const passed = `${successes}`.padStart(6); - const failed = `${failures}`.padStart(6); - let pct = `${Math.ceil(((totalPaths - paths.length) / totalPaths) * 100)}`; - pct = pct.padStart(3); - - let line = `[${mins}:${secs}|%${pct}|+${passed}|-${failed}]: ${curPath}`; - - // Truncate line like cpplint does in case it gets too long - if (line.length > 75) - line = `${line.slice(0, 75)}...`; - - // Store the line length so we know how much to erase the next time around - lastLineLen = line.length; - - outFn(line); - } -} else { - // Worker - - let config = {}; - process.on('message', (files) => { - if (files instanceof Array) { - // Lint some files - const report = cli.executeOnFiles(files); - - // If we were asked to fix the fixable issues, do so. - if (cliOptions.fix) - CLIEngine.outputFixes(report); - - if (config.sendAll) { - // Return both success and error results - - const results = report.results; - // Silence warnings for files with no errors while keeping the "ok" - // status - if (report.warningCount > 0) { - for (let i = 0; i < results.length; ++i) { - const result = results[i]; - if (result.errorCount === 0 && result.warningCount > 0) { - result.warningCount = 0; - result.messages = []; - } - } - } - process.send({ results: results, errorCount: report.errorCount }); - } else if (report.errorCount === 0) { - // No errors, return number of successful lint operations - process.send(files.length); - } else { - // One or more errors, return the error results only - process.send(CLIEngine.getErrorResults(report.results)); - } - } else if (typeof files === 'object') { - // The master process is actually sending us our configuration and not a - // list of files to lint - config = files; - } - }); -} diff --git a/vcbuild.bat b/vcbuild.bat index eb573d31b8f011..d569a515477d33 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -118,8 +118,6 @@ if /i "%1"=="test-v8-all" set test_v8=1&set test_v8_intl=1&set test_v8_ben if /i "%1"=="lint-cpp" set lint_cpp=1&goto arg-ok if /i "%1"=="lint-js" set lint_js=1&goto arg-ok if /i "%1"=="jslint" set lint_js=1&echo Please use lint-js instead of jslint&goto arg-ok -if /i "%1"=="lint-js-ci" set lint_js_ci=1&goto arg-ok -if /i "%1"=="jslint-ci" set lint_js_ci=1&echo Please use lint-js-ci instead of jslint-ci&goto arg-ok if /i "%1"=="lint-md" set lint_md=1&goto arg-ok if /i "%1"=="lint-md-build" set lint_md_build=1&goto arg-ok if /i "%1"=="lint" set lint_cpp=1&set lint_js=1&set lint_md=1&goto arg-ok @@ -655,18 +653,12 @@ goto lint-js goto lint-js :lint-js -if defined lint_js_ci goto lint-js-ci if not defined lint_js goto lint-md-build if not exist tools\node_modules\eslint goto no-lint echo running lint-js %node_exe% tools\node_modules\eslint\bin\eslint.js --cache --report-unused-disable-directives --rule "linebreak-style: 0" --ext=.js,.mjs,.md .eslintrc.js benchmark doc lib test tools goto lint-md-build -:lint-js-ci -echo running lint-js-ci -%node_exe% tools\lint-js.js -J -f tap -o test-eslint.tap benchmark doc lib test tools -goto lint-md-build - :no-lint echo Linting is not available through the source tarball. echo Use the git repo instead: $ git clone https://github.com/nodejs/node.git @@ -701,7 +693,7 @@ set exit_code=1 goto exit :help -echo vcbuild.bat [debug/release] [msi] [doc] [test/test-all/test-addons/test-js-native-api/test-node-api/test-benchmark/test-internet/test-pummel/test-simple/test-message/test-tick-processor/test-known-issues/test-node-inspect/test-check-deopts/test-npm/test-async-hooks/test-v8/test-v8-intl/test-v8-benchmarks/test-v8-all] [ignore-flaky] [static/dll] [noprojgen] [projgen] [small-icu/full-icu/without-intl] [nobuild] [nosnapshot] [noetw] [ltcg] [licensetf] [sign] [ia32/x86/x64/arm64] [vs2017/vs2019] [download-all] [lint/lint-ci/lint-js/lint-js-ci/lint-md] [lint-md-build] [package] [build-release] [upload] [no-NODE-OPTIONS] [link-module path-to-module] [debug-http2] [debug-nghttp2] [clean] [cctest] [no-cctest] [openssl-no-asm] +echo vcbuild.bat [debug/release] [msi] [doc] [test/test-all/test-addons/test-js-native-api/test-node-api/test-benchmark/test-internet/test-pummel/test-simple/test-message/test-tick-processor/test-known-issues/test-node-inspect/test-check-deopts/test-npm/test-async-hooks/test-v8/test-v8-intl/test-v8-benchmarks/test-v8-all] [ignore-flaky] [static/dll] [noprojgen] [projgen] [small-icu/full-icu/without-intl] [nobuild] [nosnapshot] [noetw] [ltcg] [licensetf] [sign] [ia32/x86/x64/arm64] [vs2017/vs2019] [download-all] [lint/lint-ci/lint-js/lint-md] [lint-md-build] [package] [build-release] [upload] [no-NODE-OPTIONS] [link-module path-to-module] [debug-http2] [debug-nghttp2] [clean] [cctest] [no-cctest] [openssl-no-asm] echo Examples: echo vcbuild.bat : builds release build echo vcbuild.bat debug : builds debug build From a7a564b4188b3ba093aa38bb5efb98e5d9e4f3ae Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Fri, 12 Jun 2020 09:20:48 -0500 Subject: [PATCH 053/492] doc: util.debuglog callback PR-URL: https://github.com/nodejs/node/pull/33856 Reviewed-By: James M Snell --- doc/api/util.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/api/util.md b/doc/api/util.md index e0860da5512de8..8633ec82c0d017 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -70,13 +70,15 @@ callbackFunction((err, ret) => { }); ``` -## `util.debuglog(section)` +## `util.debuglog(section[, callback])` * `section` {string} A string identifying the portion of the application for which the `debuglog` function is being created. +* `callback` {Function} A callback invoked the first time the logging function +is called with a function argument that is a more optimized logging function. * Returns: {Function} The logging function The `util.debuglog()` method is used to create a function that conditionally @@ -121,6 +123,19 @@ FOO-BAR 3257: hi there, it's foo-bar [2333] Multiple comma-separated `section` names may be specified in the `NODE_DEBUG` environment variable: `NODE_DEBUG=fs,net,tls`. +The optional `callback` argument can be used to replace the logging function +with a different function that doesn't have any initialization or +unnecessary wrapping. + +```js +const util = require('util'); +let debuglog = util.debuglog('internals', (debug) => { + // Replace with a logging function that optimizes out + // testing if the section is enabled + debuglog = debug; +}); +``` + ## `util.deprecate(fn, msg[, code])` + +The `'ready'` event is emitted when the `Http2Stream` has been opened, has +been assigned an `id`, and can be used. The listener does not expect any +arguments. + #### Event: `'timeout'` -In Node.js, `Buffer` objects are used to represent binary data in the form -of a sequence of bytes. Many Node.js APIs, for example streams and file system -operations, support `Buffer`s, as interactions with the operating system or -other processes generally always happen in terms of binary data. +`Buffer` objects are used to represent a fixed-length sequence of bytes. Many +Node.js APIs support `Buffer`s. -The `Buffer` class is a subclass of the [`Uint8Array`][] class that is built -into the JavaScript language. A number of additional methods are supported -that cover additional use cases. Node.js APIs accept plain [`Uint8Array`][]s -wherever `Buffer`s are supported as well. - -Instances of `Buffer`, and instances of [`Uint8Array`][] in general, -are similar to arrays of integers from `0` to `255`, but correspond to -fixed-sized blocks of memory and cannot contain any other values. -The size of a `Buffer` is established when it is created and cannot be changed. +The `Buffer` class is a subclass of JavaScript's [`Uint8Array`][] class and +extends it with methods that cover additional use cases. Node.js APIs accept +plain [`Uint8Array`][]s wherever `Buffer`s are supported as well. The `Buffer` class is within the global scope, making it unlikely that one would need to ever use `require('buffer').Buffer`. @@ -165,11 +157,10 @@ changes: description: The `Buffer`s class now inherits from `Uint8Array`. --> -`Buffer` instances are also [`Uint8Array`][] instances, which is the language’s -built-in class for working with binary data. [`Uint8Array`][] in turn is a -subclass of [`TypedArray`][]. Therefore, all [`TypedArray`][] methods are also -available on `Buffer`s. However, there are subtle incompatibilities between -the `Buffer` API and the [`TypedArray`][] API. +`Buffer` instances are also JavaScript [`Uint8Array`][] and [`TypedArray`][] +instances. All [`TypedArray`][] methods are available on `Buffer`s. There are, +however, subtle incompatibilities between the `Buffer` API and the +[`TypedArray`][] API. In particular: @@ -182,26 +173,37 @@ In particular: * [`buf.toString()`][] is incompatible with its `TypedArray` equivalent. * A number of methods, e.g. [`buf.indexOf()`][], support additional arguments. -There are two ways to create new [`TypedArray`][] instances from a `Buffer`. +There are two ways to create new [`TypedArray`][] instances from a `Buffer`: + +* Passing a `Buffer` to a [`TypedArray`][] constructor will copy the `Buffer`s + contents, interpreted an array array of integers, and not as a byte sequence + of the target type. + +```js +const buf = Buffer.from([1, 2, 3, 4]); +const uint32array = new Uint32Array(buf); + +console.log(uint32array); -When passing a `Buffer` to a [`TypedArray`][] constructor, the `Buffer`’s -elements will be copied, interpreted as an array of integers, and not as a byte -array of the target type. For example, -`new Uint32Array(Buffer.from([1, 2, 3, 4]))` creates a 4-element -[`Uint32Array`][] with elements `[1, 2, 3, 4]`, rather than a -[`Uint32Array`][] with a single element `[0x1020304]` or `[0x4030201]`. +// Prints: Uint32Array(4) [ 1, 2, 3, 4 ] +``` -In order to create a [`TypedArray`][] that shares its memory with the `Buffer`, -the underlying [`ArrayBuffer`][] can be passed to the [`TypedArray`][] -constructor instead: +* Passing the `Buffer`s underlying [`ArrayBuffer`][] will create a + [`TypedArray`][] that shares its memory with the `Buffer`. ```js const buf = Buffer.from('hello', 'utf16le'); const uint16arr = new Uint16Array( - buf.buffer, buf.byteOffset, buf.length / Uint16Array.BYTES_PER_ELEMENT); + buf.buffer, + buf.byteOffset, + buf.length / Uint16Array.BYTES_PER_ELEMENT); + +console.log(uint16array); + +// Prints: Uint16Array(5) [ 104, 101, 108, 108, 111 ] ``` -It is also possible to create a new `Buffer` that shares the same allocated +It is possible to create a new `Buffer` that shares the same allocated memory as a [`TypedArray`][] instance by using the `TypedArray` object’s `.buffer` property in the same way. [`Buffer.from()`][`Buffer.from(arrayBuf)`] behaves like `new Uint8Array()` in this context. @@ -214,6 +216,7 @@ arr[1] = 4000; // Copies the contents of `arr`. const buf1 = Buffer.from(arr); + // Shares memory with `arr`. const buf2 = Buffer.from(arr.buffer); @@ -773,7 +776,7 @@ range is between `0x00` and `0xFF` (hex) or `0` and `255` (decimal). This operator is inherited from `Uint8Array`, so its behavior on out-of-bounds access is the same as `Uint8Array`. In other words, `buf[index]` returns -`undefined` when `index` is negative or `>= buf.length`, and +`undefined` when `index` is negative or greater or equal to `buf.length`, and `buf[index] = value` does not modify the buffer if `index` is negative or `>= buf.length`. @@ -795,8 +798,8 @@ console.log(buf.toString('utf8')); ### `buf.buffer` -* {ArrayBuffer} The underlying `ArrayBuffer` object based on - which this `Buffer` object is created. +* {ArrayBuffer} The underlying `ArrayBuffer` object based on which this `Buffer` + object is created. This `ArrayBuffer` is not guaranteed to correspond exactly to the original `Buffer`. See the notes on `buf.byteOffset` for details. @@ -811,16 +814,15 @@ console.log(buffer.buffer === arrayBuffer); ### `buf.byteOffset` -* {integer} The `byteOffset` on the underlying `ArrayBuffer` object based on - which this `Buffer` object is created. +* {integer} The `byteOffset` of the `Buffer`s underlying `ArrayBuffer` object. When setting `byteOffset` in `Buffer.from(ArrayBuffer, byteOffset, length)`, -or sometimes when allocating a buffer smaller than `Buffer.poolSize`, the -buffer doesn't start from a zero offset on the underlying `ArrayBuffer`. +or sometimes when allocating a `Buffer` smaller than `Buffer.poolSize`, the +buffer does not start from a zero offset on the underlying `ArrayBuffer`. This can cause problems when accessing the underlying `ArrayBuffer` directly using `buf.buffer`, as other parts of the `ArrayBuffer` may be unrelated -to the `buf` object itself. +to the `Buffer` object itself. A common issue when creating a `TypedArray` object that shares its memory with a `Buffer` is that in this case one needs to specify the `byteOffset` correctly: @@ -1340,6 +1342,21 @@ deprecated: v8.0.0 The `buf.parent` property is a deprecated alias for `buf.buffer`. ### `buf.readBigInt64BE([offset])` + + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {bigint} + +Reads a signed, big-endian 64-bit integer from `buf` at the specified `offset`. + +Integers read from a `Buffer` are interpreted as two's complement signed +values. + ### `buf.readBigInt64LE([offset])` @@ -1365,22 +1381,38 @@ added: v12.0.0 satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. * Returns: {bigint} -Reads an unsigned 64-bit integer from `buf` at the specified `offset` with -the specified [endianness][] (`readBigUInt64BE()` reads as big endian, -`readBigUInt64LE()` reads as little endian). +Reads an unsigned, big-endian 64-bit integer from `buf` at the specified +`offset`. ```js const buf = Buffer.from([0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff]); console.log(buf.readBigUInt64BE(0)); // Prints: 4294967295n +``` + +### `buf.readBigUInt64LE([offset])` + + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {bigint} + +Reads an unsigned, little-endian 64-bit integer from `buf` at the specified +`offset`. + +```js +const buf = Buffer.from([0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff]); console.log(buf.readBigUInt64LE(0)); // Prints: 18446744069414584320n ``` ### `buf.readDoubleBE([offset])` -### `buf.readDoubleLE([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {number} + +Reads a 64-bit, little-endian double from `buf` at the specified `offset`. + +```js +const buf = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); + console.log(buf.readDoubleLE(0)); // Prints: 5.447603722011605e-270 console.log(buf.readDoubleLE(1)); @@ -1410,7 +1461,6 @@ console.log(buf.readDoubleLE(1)); ``` ### `buf.readFloatBE([offset])` -### `buf.readFloatLE([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {number} + +Reads a 32-bit, little-endian float from `buf` at the specified `offset`. + +```js +const buf = Buffer.from([1, 2, 3, 4]); + console.log(buf.readFloatLE(0)); // Prints: 1.539989614439558e-36 console.log(buf.readFloatLE(1)); @@ -1469,7 +1538,6 @@ console.log(buf.readInt8(2)); ``` ### `buf.readInt16BE([offset])` -### `buf.readInt16LE([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 2`. **Default:** `0`. +* Returns: {integer} + +Reads a signed, little-endian 16-bit integer from `buf` at the specified +`offset`. + +Integers read from a `Buffer` are interpreted as two's complement signed values. + +```js +const buf = Buffer.from([0, 5]); + console.log(buf.readInt16LE(0)); // Prints: 1280 console.log(buf.readInt16LE(1)); @@ -1501,7 +1591,6 @@ console.log(buf.readInt16LE(1)); ``` ### `buf.readInt32BE([offset])` -### `buf.readInt32LE([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {integer} + +Reads a signed, little-endian 32-bit integer from `buf` at the specified +`offset`. + +Integers read from a `Buffer` are interpreted as two's complement signed values. + +```js +const buf = Buffer.from([0, 0, 0, 5]); + console.log(buf.readInt32LE(0)); // Prints: 83886080 console.log(buf.readInt32LE(1)); @@ -1533,7 +1644,6 @@ console.log(buf.readInt32LE(1)); ``` ### `buf.readIntBE(offset, byteLength)` -### `buf.readIntLE(offset, byteLength)` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - byteLength`. +* `byteLength` {integer} Number of bytes to read. Must satisfy + `0 < byteLength <= 6`. +* Returns: {integer} + +Reads `byteLength` number of bytes from `buf` at the specified `offset` +and interprets the result as a little-endian, two's complement signed value +supporting up to 48 bits of accuracy. + +```js +const buf = Buffer.from([0x12, 0x34, 0x56, 0x78, 0x90, 0xab]); + +console.log(buf.readIntLE(0, 6).toString(16)); +// Prints: -546f87a9cbee +``` + ### `buf.readUInt8([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 2`. **Default:** `0`. +* Returns: {integer} + +Reads an unsigned, little-endian 16-bit integer from `buf` at the specified +`offset`. + +```js +const buf = Buffer.from([0x12, 0x34, 0x56]); + +console.log(buf.readUInt16LE(0).toString(16)); +// Prints: 3412 console.log(buf.readUInt16LE(1).toString(16)); // Prints: 5634 console.log(buf.readUInt16LE(2).toString(16)); @@ -1628,7 +1783,6 @@ console.log(buf.readUInt16LE(2).toString(16)); ``` ### `buf.readUInt32BE([offset])` -### `buf.readUInt32LE([offset])` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {integer} + +Reads an unsigned, little-endian 32-bit integer from `buf` at the specified +`offset`. + +```js +const buf = Buffer.from([0x12, 0x34, 0x56, 0x78]); + console.log(buf.readUInt32LE(0).toString(16)); // Prints: 78563412 console.log(buf.readUInt32LE(1).toString(16)); @@ -1658,7 +1833,6 @@ console.log(buf.readUInt32LE(1).toString(16)); ``` ### `buf.readUIntBE(offset, byteLength)` -### `buf.readUIntLE(offset, byteLength)` + +* `offset` {integer} Number of bytes to skip before starting to read. Must + satisfy `0 <= offset <= buf.length - byteLength`. +* `byteLength` {integer} Number of bytes to read. Must satisfy + `0 < byteLength <= 6`. +* Returns: {integer} + +Reads `byteLength` number of bytes from `buf` at the specified `offset` +and interprets the result as an unsigned, little-endian integer supporting +up to 48 bits of accuracy. + +```js +const buf = Buffer.from([0x12, 0x34, 0x56, 0x78, 0x90, 0xab]); + +console.log(buf.readUIntLE(0, 6).toString(16)); +// Prints: ab9078563412 +``` + ### `buf.subarray([start[, end]])` @@ -2036,9 +2234,7 @@ added: v12.0.0 satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. * Returns: {integer} `offset` plus the number of bytes written. -Writes `value` to `buf` at the specified `offset` with the specified -[endianness][] (`writeBigInt64BE()` writes as big endian, `writeBigInt64LE()` -writes as little endian). +Writes `value` to `buf` at the specified `offset` as big-endian. `value` is interpreted and written as a two's complement signed integer. @@ -2051,7 +2247,54 @@ console.log(buf); // Prints: ``` +### `buf.writeBigInt64LE(value[, offset])` + + +* `value` {bigint} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. + +`value` is interpreted and written as a two's complement signed integer. + +```js +const buf = Buffer.allocUnsafe(8); + +buf.writeBigInt64LE(0x0102030405060708n, 0); + +console.log(buf); +// Prints: +``` + ### `buf.writeBigUInt64BE(value[, offset])` + + +* `value` {bigint} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy: `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as big-endian. + +```js +const buf = Buffer.allocUnsafe(8); + +buf.writeBigUInt64BE(0xdecafafecacefaden, 0); + +console.log(buf); +// Prints: +``` + ### `buf.writeBigUInt64LE(value[, offset])` + +* `value` {number} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 8`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. The `value` +must be a JavaScript number. Behavior is undefined when `value` is anything +other than a JavaScript number. + +```js +const buf = Buffer.allocUnsafe(8); buf.writeDoubleLE(123.456, 0); @@ -2111,7 +2373,6 @@ console.log(buf); ``` ### `buf.writeFloatBE(value[, offset])` -### `buf.writeFloatLE(value[, offset])` + +* `value` {number} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. Behavior is +undefined when `value` is anything other than a JavaScript number. + +```js +const buf = Buffer.allocUnsafe(4); + buf.writeFloatLE(0xcafebabe, 0); console.log(buf); @@ -2177,7 +2459,6 @@ console.log(buf); ``` ### `buf.writeInt16BE(value[, offset])` -### `buf.writeInt16LE(value[, offset])` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 2`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. The `value` +must be a valid signed 16-bit integer. Behavior is undefined when `value` is +anything other than a signed 16-bit integer. + +The `value` is interpreted and written as a two's complement signed integer. + +```js +const buf = Buffer.allocUnsafe(2); + +buf.writeInt16LE(0x0304, 0); + +console.log(buf); +// Prints: ``` ### `buf.writeInt32BE(value[, offset])` -### `buf.writeInt32LE(value[, offset])` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. The `value` +must be a valid signed 32-bit integer. Behavior is undefined when `value` is +anything other than a signed 32-bit integer. + +The `value` is interpreted and written as a two's complement signed integer. + +```js +const buf = Buffer.allocUnsafe(4); + +buf.writeInt32LE(0x05060708, 0); + +console.log(buf); +// Prints: ``` ### `buf.writeIntBE(value, offset, byteLength)` -### `buf.writeIntLE(value, offset, byteLength)` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - byteLength`. +* `byteLength` {integer} Number of bytes to write. Must satisfy + `0 < byteLength <= 6`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `byteLength` bytes of `value` to `buf` at the specified `offset` +as little-endian. Supports up to 48 bits of accuracy. Behavior is undefined +when `value` is anything other than a signed integer. + +```js +const buf = Buffer.allocUnsafe(6); + buf.writeIntLE(0x1234567890ab, 0, 6); console.log(buf); @@ -2310,7 +2671,6 @@ console.log(buf); ``` ### `buf.writeUInt16BE(value[, offset])` -### `buf.writeUInt16LE(value[, offset])` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 2`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. The `value` +must be a valid unsigned 16-bit integer. Behavior is undefined when `value` is +anything other than an unsigned 16-bit integer. + +```js +const buf = Buffer.allocUnsafe(4); buf.writeUInt16LE(0xdead, 0); buf.writeUInt16LE(0xbeef, 2); @@ -2347,7 +2729,6 @@ console.log(buf); ``` ### `buf.writeUInt32BE(value[, offset])` -### `buf.writeUInt32LE(value[, offset])` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - 4`. **Default:** `0`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `value` to `buf` at the specified `offset` as little-endian. The `value` +must be a valid unsigned 32-bit integer. Behavior is undefined when `value` is +anything other than an unsigned 32-bit integer. + +```js +const buf = Buffer.allocUnsafe(4); buf.writeUInt32LE(0xfeedface, 0); @@ -2382,7 +2785,6 @@ console.log(buf); ``` ### `buf.writeUIntBE(value, offset, byteLength)` -### `buf.writeUIntLE(value, offset, byteLength)` + +* `value` {integer} Number to be written to `buf`. +* `offset` {integer} Number of bytes to skip before starting to write. Must + satisfy `0 <= offset <= buf.length - byteLength`. +* `byteLength` {integer} Number of bytes to write. Must satisfy + `0 < byteLength <= 6`. +* Returns: {integer} `offset` plus the number of bytes written. + +Writes `byteLength` bytes of `value` to `buf` at the specified `offset` +as little-endian. Supports up to 48 bits of accuracy. Behavior is undefined +when `value` is anything other than an unsigned integer. + +```js +const buf = Buffer.allocUnsafe(6); buf.writeUIntLE(0x1234567890ab, 0, 6); @@ -2547,7 +2974,13 @@ changes: See [`Buffer.from(string[, encoding])`][`Buffer.from(string)`]. -## `buffer.INSPECT_MAX_BYTES` +## `buffer` module APIs + +While, the `Buffer` object is available as a global, there are additional +`Buffer`-related APIs that are available only via the `buffer` module +accessed using `require('buffer')`. + +### `buffer.INSPECT_MAX_BYTES` @@ -2558,10 +2991,7 @@ Returns the maximum number of bytes that will be returned when `buf.inspect()` is called. This can be overridden by user modules. See [`util.inspect()`][] for more details on `buf.inspect()` behavior. -This is a property on the `buffer` module returned by -`require('buffer')`, not on the `Buffer` global or a `Buffer` instance. - -## `buffer.kMaxLength` +### `buffer.kMaxLength` @@ -2570,10 +3000,7 @@ added: v3.0.0 An alias for [`buffer.constants.MAX_LENGTH`][]. -This is a property on the `buffer` module returned by -`require('buffer')`, not on the `Buffer` global or a `Buffer` instance. - -## `buffer.transcode(source, fromEnc, toEnc)` +### `buffer.transcode(source, fromEnc, toEnc)` @@ -2624,7 +3048,7 @@ See [`Buffer.allocUnsafeSlow()`][]. This was never a class in the sense that the constructor always returned a `Buffer` instance, rather than a `SlowBuffer` instance. -### `new SlowBuffer(size)` +#### `new SlowBuffer(size)` @@ -2635,15 +3059,12 @@ deprecated: v6.0.0 See [`Buffer.allocUnsafeSlow()`][]. -## Buffer constants +### Buffer constants -`buffer.constants` is a property on the `buffer` module returned by -`require('buffer')`, not on the `Buffer` global or a `Buffer` instance. - -### `buffer.constants.MAX_LENGTH` +#### `buffer.constants.MAX_LENGTH` @@ -2655,7 +3076,7 @@ On 64-bit architectures, this value currently is `(2^31)-1` (~2GB). This value is also available as [`buffer.kMaxLength`][]. -### `buffer.constants.MAX_STRING_LENGTH` +#### `buffer.constants.MAX_STRING_LENGTH` @@ -2799,7 +3220,6 @@ introducing security vulnerabilities into an application. [`TypedArray#slice()`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/slice [`TypedArray#subarray()`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/subarray [`TypedArray`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray -[`Uint32Array`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint32Array [`Uint8Array`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array [`buf.buffer`]: #buffer_buf_buffer [`buf.compare()`]: #buffer_buf_compare_target_targetstart_targetend_sourcestart_sourceend From 2ac653dc1a37479e0ecbad8a896fff9dc8c0ef63 Mon Sep 17 00:00:00 2001 From: Derek Lewis Date: Tue, 7 Jul 2020 11:01:11 -0400 Subject: [PATCH 087/492] meta: make issue template mobile friendly and address nits PR-URL: https://github.com/nodejs/node/pull/34243 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Evan Lucas Reviewed-By: Trivikram Kamat Reviewed-By: Rich Trott Reviewed-By: Anto Aravinth --- .../ISSUE_TEMPLATE/3-api-ref-docs-problem.md | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/3-api-ref-docs-problem.md b/.github/ISSUE_TEMPLATE/3-api-ref-docs-problem.md index 3ec3959eb67173..f63d540abaf4e2 100644 --- a/.github/ISSUE_TEMPLATE/3-api-ref-docs-problem.md +++ b/.github/ISSUE_TEMPLATE/3-api-ref-docs-problem.md @@ -7,29 +7,48 @@ labels: doc # 📗 API Reference Docs Problem - +For more general support, please open an issue +using the issue tracker for our help repository. + + https://github.com/nodejs/help + +--- + +For the issue title, please enter a one-line +summary after “doc: ” (preferably 50 characters +or less and no more than 72). + +The “✍️” are placeholders signifying requests for +input. Replace them with your responses. + +If you are unsure of something, do your best. - + + - **Version**: ✍️ + + + - **Platform**: ✍️ + + + - **Subsystem**: ✍️ ## Location @@ -37,18 +56,22 @@ Subsystem: if known, please specify affected core module name _Section of the site where the content exists_ Affected URL(s): + - https://nodejs.org/api/✍️ -## Problem description +## Description -_Concise explanation of what you found to be problematic_ +_Concise explanation of the problem_ - + ✍️ --- - + -- [ ] I would like to work on this issue and submit a pull request. +- [ ] I would like to work on this issue and + submit a pull request. From 97415103366a36e30bc50704ef98218f274b9afe Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Sat, 11 Jul 2020 18:12:18 -0700 Subject: [PATCH 088/492] test: fix flaky test-http2-reset-flood Set `allowHalfOpen: true` in the client. Fixes: https://github.com/nodejs/node/issues/29802 Refs: https://github.com/nodejs/node/pull/31806 PR-URL: https://github.com/nodejs/node/pull/34318 Reviewed-By: Anna Henningsen Reviewed-By: Robert Nagy --- test/parallel/parallel.status | 4 ---- test/parallel/test-http2-reset-flood.js | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/test/parallel/parallel.status b/test/parallel/parallel.status index d95b3e45d40252..b50684d96fcc7e 100644 --- a/test/parallel/parallel.status +++ b/test/parallel/parallel.status @@ -5,8 +5,6 @@ prefix parallel # sample-test : PASS,FLAKY [true] # This section applies to all platforms -# https://github.com/nodejs/node/issues/29802 -test-http2-reset-flood: PASS,FLAKY [$system==win32] # https://github.com/nodejs/node/issues/32863 @@ -48,8 +46,6 @@ test-fs-stream-construct: PASS,FLAKY [$system==freebsd] # https://github.com/nodejs/node/issues/31727 test-fs-stat-bigint: PASS,FLAKY -# https://github.com/nodejs/node/issues/29802 -test-http2-reset-flood: PASS,FLAKY # https://github.com/nodejs/node/issues/28803 test-stdout-close-catch: PASS,FLAKY # https://github.com/nodejs/node/issues/31280 diff --git a/test/parallel/test-http2-reset-flood.js b/test/parallel/test-http2-reset-flood.js index d721384ab3742e..db61b7d9292ce6 100644 --- a/test/parallel/test-http2-reset-flood.js +++ b/test/parallel/test-http2-reset-flood.js @@ -28,7 +28,7 @@ if (process.env.HAS_STARTED_WORKER) { process.env.HAS_STARTED_WORKER = 1; const worker = new Worker(__filename).on('message', common.mustCall((port) => { const h2header = Buffer.alloc(9); - const conn = net.connect(port); + const conn = net.connect({ port, allowHalfOpen: true }); conn.write('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'); From a2107101be97c88f6aa5c4dbcf46f5acffede90e Mon Sep 17 00:00:00 2001 From: Danielle Adams Date: Tue, 14 Jul 2020 09:32:24 -0400 Subject: [PATCH 089/492] doc: add danielleadams to collaborators PR-URL: https://github.com/nodejs/node/pull/34360 Reviewed-By: Anna Henningsen Reviewed-By: Rich Trott Reviewed-By: Gireesh Punathil --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8ef40a4d56fff8..85211650f71096 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ For information about the governance of the Node.js project, see **Shelley Vohr** <codebytere@gmail.com> (she/her) * [danbev](https://github.com/danbev) - **Daniel Bevenius** <daniel.bevenius@gmail.com> (he/him) +* [danielleadams](https://github.com/danielleadams) - +**Danielle Adams** <adamzdanielle@gmail.com> (she/her) * [davisjam](https://github.com/davisjam) - **Jamie Davis** <davisjam@vt.edu> (he/him) * [devnexen](https://github.com/devnexen) - From c1b5c89e60b618c14e60ff4b03009e0605ce8e17 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Wed, 8 Jul 2020 22:32:13 -0700 Subject: [PATCH 090/492] doc: reword warnings about sockets passed to subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the docs more concise. Make warnings direct ("do not use" and "should") rather than "is not recommended" or "is recommended". Backport-PR-URL: https://github.com/nodejs/node/pull/34377 PR-URL: https://github.com/nodejs/node/pull/34273 Reviewed-By: Tobias Nießen Reviewed-By: James M Snell Reviewed-By: Anto Aravinth Reviewed-By: Luigi Pinca Reviewed-By: Trivikram Kamat --- doc/api/child_process.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/api/child_process.md b/doc/api/child_process.md index ad9d42d9c74452..07a08d50869bb6 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -1356,14 +1356,12 @@ process.on('message', (m, socket) => { }); ``` -Once a socket has been passed to a child, the parent is no longer capable of -tracking when the socket is destroyed. To indicate this, the `.connections` -property becomes `null`. It is recommended not to use `.maxConnections` when -this occurs. - -It is also recommended that any `'message'` handlers in the child process -verify that `socket` exists, as the connection may have been closed during the -time it takes to send the connection to the child. +Do not use `.maxConnections` on a socket that has been passed to a subprocess. +The parent cannot track when the socket is destroyed. + +Any `'message'` handlers in the subprocess should verify that `socket` exists, +as the connection may have been closed during the time it takes to send the +connection to the child. ### `subprocess.signalCode` From 7be8dded526be29b2ae65cc6befb1238c6d350d7 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 10 Jul 2020 23:37:20 -0700 Subject: [PATCH 091/492] doc: clarify conditional exports guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/34306 Reviewed-By: Rich Trott Reviewed-By: James M Snell Reviewed-By: Michaël Zasso Reviewed-By: Jan Krems --- doc/api/esm.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 0691a5cce3d5da..254d7240a733b5 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -456,7 +456,7 @@ Conditional exports can also be extended to exports subpaths, for example: "exports": { ".": "./main.js", "./feature": { - "browser": "./feature-browser.js", + "node": "./feature-node.js", "default": "./feature.js" } } @@ -464,8 +464,16 @@ Conditional exports can also be extended to exports subpaths, for example: ``` Defines a package where `require('pkg/feature')` and `import 'pkg/feature'` -could provide different implementations between the browser and Node.js, -given third-party tool support for a `"browser"` condition. +could provide different implementations between Node.js and other JS +environments. + +When using environment branches, always include a `"default"` condition where +possible. Providing a `"default"` condition ensures that any unknown JS +environments are able to use this universal implementation, which helps avoid +these JS environments from having to pretend to be existing environments in +order to support packages with conditional exports. For this reason, using +`"node"` and `"default"` condition branches is usually preferable to using +`"node"` and `"browser"` condition branches. #### Nested conditions @@ -479,11 +487,11 @@ use in Node.js but not the browser: { "main": "./main.js", "exports": { - "browser": "./feature-browser.mjs", "node": { "import": "./feature-node.mjs", "require": "./feature-node.cjs" - } + }, + "default": "./feature.mjs", } } ``` From 0ede223fa83c69dc14cea43dfef9f3e617a44c25 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Wed, 11 Sep 2019 10:03:11 -0500 Subject: [PATCH 092/492] policy: add startup benchmark and make SRI lazier PR-URL: https://github.com/nodejs/node/pull/29527 Reviewed-By: Joyee Cheung Reviewed-By: James M Snell --- benchmark/policy/policy-startup.js | 51 ++++++++++++++++ lib/internal/policy/manifest.js | 78 ++++++++++++++++--------- lib/internal/policy/sri.js | 14 +++-- test/benchmark/test-benchmark-policy.js | 9 +++ 4 files changed, 119 insertions(+), 33 deletions(-) create mode 100644 benchmark/policy/policy-startup.js create mode 100644 test/benchmark/test-benchmark-policy.js diff --git a/benchmark/policy/policy-startup.js b/benchmark/policy/policy-startup.js new file mode 100644 index 00000000000000..1588123d8007d9 --- /dev/null +++ b/benchmark/policy/policy-startup.js @@ -0,0 +1,51 @@ +// Tests the impact on eager operations required for policies affecting +// general startup, does not test lazy operations +'use strict'; +const common = require('../common.js'); + +const configs = { + n: [1024] +}; + +const options = { + flags: ['--expose-internals'] +}; + +const bench = common.createBenchmark(main, configs, options); + +function main(conf) { + const hash = (str, algo) => { + const hash = require('crypto').createHash(algo); + return hash.update(str).digest('base64'); + }; + const resources = Object.fromEntries( + // Simulate graph of 1k modules + Array.from({ length: 1024 }, (_, i) => { + return [`./_${i}`, { + integrity: `sha256-${hash(`// ./_${i}`, 'sha256')}`, + dependencies: Object.fromEntries(Array.from({ + // Average 3 deps per 4 modules + length: Math.floor((i % 4) / 2) + }, (_, ii) => { + return [`_${ii}`, `./_${i - ii}`]; + })), + }]; + }) + ); + const json = JSON.parse(JSON.stringify({ resources }), (_, o) => { + if (o && typeof o === 'object') { + Reflect.setPrototypeOf(o, null); + Object.freeze(o); + } + return o; + }); + const { Manifest } = require('internal/policy/manifest'); + + bench.start(); + + for (let i = 0; i < conf.n; i++) { + new Manifest(json, 'file://benchmark/policy-relative'); + } + + bench.end(conf.n); +} diff --git a/lib/internal/policy/manifest.js b/lib/internal/policy/manifest.js index ae5e6cd3f43dfd..7283606478d383 100644 --- a/lib/internal/policy/manifest.js +++ b/lib/internal/policy/manifest.js @@ -4,6 +4,7 @@ const { ArrayIsArray, Map, MapPrototypeSet, + ObjectCreate, ObjectEntries, ObjectFreeze, ObjectSetPrototypeOf, @@ -29,7 +30,6 @@ const { URL } = require('internal/url'); const { createHash, timingSafeEqual } = crypto; const HashUpdate = uncurryThis(crypto.Hash.prototype.update); const HashDigest = uncurryThis(crypto.Hash.prototype.digest); -const BufferEquals = uncurryThis(Buffer.prototype.equals); const BufferToString = uncurryThis(Buffer.prototype.toString); const kRelativeURLStringPattern = /^\.{0,2}\//; const { getOptionValue } = require('internal/options'); @@ -54,9 +54,47 @@ function REACTION_LOG(error) { } class Manifest { + /** + * @type {Map} + * + * Used to compare a resource to the content body at the resource. + * `true` is used to signify that all integrities are allowed, otherwise, + * SRI strings are parsed to compare with the body. + * + * This stores strings instead of eagerly parsing SRI strings + * and only converts them to SRI data structures when needed. + * This avoids needing to parse all SRI strings at startup even + * if some never end up being used. + */ #integrities = new SafeMap(); + /** + * @type {Map true | URL>} + * + * Used to find where a dependency is located. + * + * This stores functions to lazily calculate locations as needed. + * `true` is used to signify that the location is not specified + * by the manifest and default resolution should be allowed. + */ #dependencies = new SafeMap(); + /** + * @type {(err: Error) => void} + * + * Performs default action for what happens when a manifest encounters + * a violation such as abort()ing or exiting the process, throwing the error, + * or logging the error. + */ #reaction = null; + /** + * `obj` should match the policy file format described in the docs + * it is expected to not have prototype pollution issues either by reassigning + * the prototype to `null` for values or by running prior to any user code. + * + * `manifestURL` is a URL to resolve relative locations against. + * + * @param {object} obj + * @param {string} manifestURL + */ constructor(obj, manifestURL) { const integrities = this.#integrities; const dependencies = this.#dependencies; @@ -96,35 +134,14 @@ class Manifest { let integrity = manifestEntries[i][1].integrity; if (!integrity) integrity = null; if (integrity != null) { - debug(`Manifest contains integrity for url ${originalHREF}`); + debug('Manifest contains integrity for url %s', originalHREF); if (typeof integrity === 'string') { - const sri = ObjectFreeze(SRI.parse(integrity)); if (integrities.has(resourceHREF)) { - const old = integrities.get(resourceHREF); - let mismatch = false; - - if (old.length !== sri.length) { - mismatch = true; - } else { - compare: - for (let sriI = 0; sriI < sri.length; sriI++) { - for (let oldI = 0; oldI < old.length; oldI++) { - if (sri[sriI].algorithm === old[oldI].algorithm && - BufferEquals(sri[sriI].value, old[oldI].value) && - sri[sriI].options === old[oldI].options) { - continue compare; - } - } - mismatch = true; - break compare; - } - } - - if (mismatch) { + if (integrities.get(resourceHREF) !== integrity) { throw new ERR_MANIFEST_INTEGRITY_MISMATCH(resourceURL); } } - integrities.set(resourceHREF, sri); + integrities.set(resourceHREF, integrity); } else if (integrity === true) { integrities.set(resourceHREF, true); } else { @@ -136,7 +153,7 @@ class Manifest { let dependencyMap = manifestEntries[i][1].dependencies; if (dependencyMap === null || dependencyMap === undefined) { - dependencyMap = {}; + dependencyMap = ObjectCreate(null); } if (typeof dependencyMap === 'object' && !ArrayIsArray(dependencyMap)) { /** @@ -198,13 +215,18 @@ class Manifest { assertIntegrity(url, content) { const href = `${url}`; - debug(`Checking integrity of ${href}`); + debug('Checking integrity of %s', href); const integrities = this.#integrities; const realIntegrities = new Map(); if (integrities.has(href)) { - const integrityEntries = integrities.get(href); + let integrityEntries = integrities.get(href); if (integrityEntries === true) return true; + if (typeof integrityEntries === 'string') { + const sri = ObjectFreeze(SRI.parse(integrityEntries)); + integrities.set(href, sri); + integrityEntries = sri; + } // Avoid clobbered Symbol.iterator for (let i = 0; i < integrityEntries.length; i++) { const { diff --git a/lib/internal/policy/sri.js b/lib/internal/policy/sri.js index d70df5c1aa1f7b..c4b5e7f1ca209a 100644 --- a/lib/internal/policy/sri.js +++ b/lib/internal/policy/sri.js @@ -1,17 +1,19 @@ 'use strict'; -// Value of https://w3c.github.io/webappsec-subresource-integrity/#the-integrity-attribute +// Utility to parse the value of +// https://w3c.github.io/webappsec-subresource-integrity/#the-integrity-attribute const { ObjectDefineProperty, ObjectFreeze, + ObjectGetPrototypeOf, ObjectSeal, + ObjectSetPrototypeOf, RegExp, RegExpPrototypeExec, RegExpPrototypeTest, StringPrototypeSlice, } = primordials; -// Returns [{algorithm, value (in base64 string), options,}] const { ERR_SRI_PARSE } = require('internal/errors').codes; @@ -21,7 +23,8 @@ const kHASH_ALGO = 'sha(?:256|384|512)'; // Base64 const kHASH_VALUE = '[A-Za-z0-9+/]+[=]{0,2}'; const kHASH_EXPRESSION = `(${kHASH_ALGO})-(${kHASH_VALUE})`; -const kOPTION_EXPRESSION = `(${kVCHAR}*)`; +// Ungrouped since unused +const kOPTION_EXPRESSION = `(?:${kVCHAR}*)`; const kHASH_WITH_OPTIONS = `${kHASH_EXPRESSION}(?:[?](${kOPTION_EXPRESSION}))?`; const kSRIPattern = RegExp(`(${kWSP}*)(?:${kHASH_WITH_OPTIONS})`, 'g'); ObjectSeal(kSRIPattern); @@ -29,9 +32,10 @@ const kAllWSP = RegExp(`^${kWSP}*$`); ObjectSeal(kAllWSP); const BufferFrom = require('buffer').Buffer.from; +const RealArrayPrototype = ObjectGetPrototypeOf([]); +// Returns {algorithm, value (in base64 string), options,}[] const parse = (str) => { - kSRIPattern.lastIndex = 0; let prevIndex = 0; let match; const entries = []; @@ -62,7 +66,7 @@ const parse = (str) => { throw new ERR_SRI_PARSE(str, str.charAt(prevIndex), prevIndex); } } - return entries; + return ObjectSetPrototypeOf(entries, RealArrayPrototype); }; module.exports = { diff --git a/test/benchmark/test-benchmark-policy.js b/test/benchmark/test-benchmark-policy.js new file mode 100644 index 00000000000000..7eb0992b1f1eda --- /dev/null +++ b/test/benchmark/test-benchmark-policy.js @@ -0,0 +1,9 @@ +'use strict'; + +require('../common'); + +const runBenchmark = require('../common/benchmark'); + +runBenchmark('policy', [ + 'n=1', +]); From bd3cef7e0fb0f72638a04e18c7cf9ae7e74f6c5b Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Sun, 12 Jul 2020 15:36:02 -0700 Subject: [PATCH 093/492] test: use mustCall() in pummel test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace 'exit' check with common.mustCall(). PR-URL: https://github.com/nodejs/node/pull/34327 Reviewed-By: Anna Henningsen Reviewed-By: Michaël Zasso Reviewed-By: Luigi Pinca Reviewed-By: James M Snell --- test/pummel/test-net-connect-econnrefused.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/pummel/test-net-connect-econnrefused.js b/test/pummel/test-net-connect-econnrefused.js index 39737f73bf0831..38a71b6cfd4944 100644 --- a/test/pummel/test-net-connect-econnrefused.js +++ b/test/pummel/test-net-connect-econnrefused.js @@ -48,17 +48,14 @@ function pummel() { } function check() { - setTimeout(function() { + setTimeout(common.mustCall(function() { assert.strictEqual(process._getActiveRequests().length, 0); const activeHandles = process._getActiveHandles(); assert.ok(activeHandles.every((val) => val.constructor.name !== 'Socket')); - check_called = true; - }, 0); + }), 0); } -let check_called = false; process.on('exit', function() { assert.strictEqual(rounds, ROUNDS); assert.strictEqual(reqs, ROUNDS * ATTEMPTS_PER_ROUND); - assert(check_called); }); From c3ac5e945caa12ad5e43a04917fa273930c10f9c Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Sun, 12 Jul 2020 17:40:50 -0700 Subject: [PATCH 094/492] test: fix flaky test-net-connect-econnrefused Test is flaky in CI with `common.PORT` but not flaky if a port is determined from createServer() first. PR-URL: https://github.com/nodejs/node/pull/34330 Reviewed-By: Richard Lau Reviewed-By: Anna Henningsen --- test/pummel/test-net-connect-econnrefused.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/pummel/test-net-connect-econnrefused.js b/test/pummel/test-net-connect-econnrefused.js index 38a71b6cfd4944..4fd4f8b6943e3c 100644 --- a/test/pummel/test-net-connect-econnrefused.js +++ b/test/pummel/test-net-connect-econnrefused.js @@ -31,12 +31,17 @@ const ATTEMPTS_PER_ROUND = 50; let rounds = 1; let reqs = 0; -pummel(); +let port; +const server = net.createServer().listen(0, common.mustCall(() => { + port = server.address().port; + server.close(common.mustCall(pummel)); +})); function pummel() { let pending; for (pending = 0; pending < ATTEMPTS_PER_ROUND; pending++) { - net.createConnection(common.PORT).on('error', function(err) { + net.createConnection(port).on('error', function(err) { + console.log('pending', pending, 'rounds', rounds); assert.strictEqual(err.code, 'ECONNREFUSED'); if (--pending > 0) return; if (rounds === ROUNDS) return check(); From 7b29c919440056be8ea9711ec131c88975564535 Mon Sep 17 00:00:00 2001 From: Francisco Ryan Tolmasky I Date: Tue, 30 Jun 2020 12:35:28 -0700 Subject: [PATCH 095/492] doc,tools: syntax highlight api docs at compile-time Prior to this commit, API docs would be highlighted after the page loaded using `highlightjs`. This commit still uses `highlightjs`, but runs the generation during the compilation of the documentation, making it so no script needs to load on the client. This results in a faster load, as well as allowing the pages to fully functional even when JavaScript is turned off. PR-URL: https://github.com/nodejs/node/pull/34148 Reviewed-By: James M Snell Reviewed-By: Anna Henningsen --- doc/api_assets/README.md | 19 ------------------- doc/api_assets/highlight.pack.js | 6 ------ doc/template.html | 4 ++-- tools/doc/allhtml.js | 6 +++--- tools/doc/html.js | 16 ++++++++++++++++ tools/doc/package-lock.json | 5 +++++ tools/doc/package.json | 1 + 7 files changed, 27 insertions(+), 30 deletions(-) delete mode 100644 doc/api_assets/highlight.pack.js diff --git a/doc/api_assets/README.md b/doc/api_assets/README.md index 3c0e1d1cbb2976..07262bba4ce01a 100644 --- a/doc/api_assets/README.md +++ b/doc/api_assets/README.md @@ -1,22 +1,5 @@ # API Reference Document Assets -## highlight.pack.js - -_Generated by [highlightjs.org/download][] on 2020-06-07._ - -Grammars included in the custom bundle: - -* Bash -* C -* C++ -* CoffeeScript -* HTTP -* JavaScript -* JSON -* Markdown -* Plaintext -* Shell Session - ## hljs.css The syntax theme for code snippets in API reference documents. @@ -24,5 +7,3 @@ The syntax theme for code snippets in API reference documents. ## style.css The main stylesheet for API reference documents. - -[highlightjs.org/download]: https://highlightjs.org/download/ diff --git a/doc/api_assets/highlight.pack.js b/doc/api_assets/highlight.pack.js deleted file mode 100644 index bd134963d24609..00000000000000 --- a/doc/api_assets/highlight.pack.js +++ /dev/null @@ -1,6 +0,0 @@ -/* - Highlight.js 10.0.3 (a4b1bd2d) - License: BSD-3-Clause - Copyright (c) 2006-2020, Ivan Sagalaev -*/ -var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!n.hasOwnProperty(r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}function n(e){return e.replace(/&/g,"&").replace(//g,">")}function t(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach((function(e){for(n in e)t[n]=e[n]})),t}function r(e){return e.nodeName.toLowerCase()}var a=Object.freeze({__proto__:null,escapeHTML:n,inherit:t,nodeStream:function(e){var n=[];return function e(t,a){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=e(i,a),r(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n},mergeStreams:function(e,t,a){var i=0,s="",o=[];function l(){return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||t.length;){var g=l();if(s+=n(a.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+n(a.substr(i))}});const i="",s=e=>!!e.kind;class o{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=n(e)}openNode(e){if(!s(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){s(e)&&(this.buffer+=i)}span(e){this.buffer+=``}value(){return this.buffer}}class l{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){let n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){e.children&&(e.children.every(e=>"string"==typeof e)?(e.text=e.children.join(""),delete e.children):e.children.forEach(e=>{"string"!=typeof e&&l._collapse(e)}))}}class c extends l{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){let t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new o(this,this.options).value()}finalize(){}}function u(e){return e&&e.source||e}const d="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",g={begin:"\\\\[\\s\\S]",relevance:0},h={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[g]},f={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[g]},p={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,r){var a=t({className:"comment",begin:e,end:n,contains:[]},r||{});return a.contains.push(p),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|XXX):",relevance:0}),a},b=m("//","$"),v=m("/\\*","\\*/"),x=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:d,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",BACKSLASH_ESCAPE:g,APOS_STRING_MODE:h,QUOTE_STRING_MODE:f,PHRASAL_WORDS_MODE:p,COMMENT:m,C_LINE_COMMENT_MODE:b,C_BLOCK_COMMENT_MODE:v,HASH_COMMENT_MODE:x,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:d,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^\/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[g,{begin:/\[/,end:/\]/,relevance:0,contains:[g]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0}}),E="of and for in not or if then".split(" ");function R(e,n){return n?+n:(t=e,E.includes(t.toLowerCase())?0:1);var t}const N=n,w=t,{nodeStream:y,mergeStreams:O}=a;return function(n){var r=[],a={},i={},s=[],o=!0,l=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,d="Could not find the language '{}', did you forget to load/include a language module?",g={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0,__emitter:c};function h(e){return g.noHighlightRe.test(e)}function f(e,n,t,r){var a={code:n,language:e};T("before:highlight",a);var i=a.result?a.result:p(a.language,a.code,t,r);return i.code=a.code,T("after:highlight",i),i}function p(e,n,r,i){var s=n;function l(e,n){var t=v.case_insensitive?n[0].toLowerCase():n[0];return e.keywords.hasOwnProperty(t)&&e.keywords[t]}function c(){null!=_.subLanguage?function(){if(""!==k){var e="string"==typeof _.subLanguage;if(!e||a[_.subLanguage]){var n=e?p(_.subLanguage,k,!0,E[_.subLanguage]):m(k,_.subLanguage.length?_.subLanguage:void 0);_.relevance>0&&(T+=n.relevance),e&&(E[_.subLanguage]=n.top),w.addSublanguage(n.emitter,n.language)}else w.addText(k)}}():function(){var e,n,t,r;if(_.keywords){for(n=0,_.lexemesRe.lastIndex=0,t=_.lexemesRe.exec(k),r="";t;){r+=k.substring(n,t.index);var a=null;(e=l(_,t))?(w.addText(r),r="",T+=e[1],a=e[0],w.addKeyword(t[0],a)):r+=t[0],n=_.lexemesRe.lastIndex,t=_.lexemesRe.exec(k)}r+=k.substr(n),w.addText(r)}else w.addText(k)}(),k=""}function h(e){e.className&&w.openNode(e.className),_=Object.create(e,{parent:{value:_}})}var f={};function b(n,t){var a,i=t&&t[0];if(k+=n,null==i)return c(),0;if("begin"==f.type&&"end"==t.type&&f.index==t.index&&""===i){if(k+=s.slice(t.index,t.index+1),!o)throw(a=Error("0 width match regex")).languageName=e,a.badRule=f.rule,a;return 1}if(f=t,"begin"===t.type)return function(e){var n=e[0],t=e.rule;return t.__onBegin&&(t.__onBegin(e)||{}).ignoreMatch?function(e){return 0===_.matcher.regexIndex?(k+=e[0],1):(B=!0,0)}(n):(t&&t.endSameAsBegin&&(t.endRe=RegExp(n.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),t.skip?k+=n:(t.excludeBegin&&(k+=n),c(),t.returnBegin||t.excludeBegin||(k=n)),h(t),t.returnBegin?0:n.length)}(t);if("illegal"===t.type&&!r)throw(a=Error('Illegal lexeme "'+i+'" for mode "'+(_.className||"")+'"')).mode=_,a;if("end"===t.type){var l=function(e){var n=e[0],t=s.substr(e.index),r=function e(n,t){if(function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(n.endRe,t)){for(;n.endsParent&&n.parent;)n=n.parent;return n}if(n.endsWithParent)return e(n.parent,t)}(_,t);if(r){var a=_;a.skip?k+=n:(a.returnEnd||a.excludeEnd||(k+=n),c(),a.excludeEnd&&(k=n));do{_.className&&w.closeNode(),_.skip||_.subLanguage||(T+=_.relevance),_=_.parent}while(_!==r.parent);return r.starts&&(r.endSameAsBegin&&(r.starts.endRe=r.endRe),h(r.starts)),a.returnEnd?0:n.length}}(t);if(null!=l)return l}if("illegal"===t.type&&""===i)return 1;if(A>1e5&&A>3*t.index)throw Error("potential infinite loop, way more iterations than matches");return k+=i,i.length}var v=M(e);if(!v)throw console.error(d.replace("{}",e)),Error('Unknown language: "'+e+'"');!function(e){function n(n,t){return RegExp(u(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class r{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);let e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+="|"),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"==l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("==l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;let n=this.matcherRe.exec(e);if(!n)return null;let t=n.findIndex((e,n)=>n>0&&null!=e),r=this.matchIndexes[t];return Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];let n=new r;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){let n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;let t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e){let n=e.input[e.index-1],t=e.input[e.index+e[0].length];if("."===n||"."===t)return{ignoreMatch:!0}}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");!function r(s,o){s.compiled||(s.compiled=!0,s.__onBegin=null,s.keywords=s.keywords||s.beginKeywords,s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,R(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemesRe=n(s.lexemes||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__onBegin=i),s.begin||(s.begin=/\B|\b/),s.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(s.endRe=n(s.end)),s.terminator_end=u(s.end)||"",s.endsWithParent&&o.terminator_end&&(s.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(s.illegalRe=n(s.illegal)),null==s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return t(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?t(e,{starts:e.starts?t(e.starts):null}):Object.isFrozen(e)?t(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){r(e,s)})),s.starts&&r(s.starts,o),s.matcher=function(e){let n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(s))}(e)}(v);var x,_=i||v,E={},w=new g.__emitter(g);!function(){for(var e=[],n=_;n!==v;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>w.openNode(e))}();var y,O,k="",T=0,L=0,A=0,B=!1;try{for(_.matcher.considerAll();A++,B?B=!1:(_.matcher.lastIndex=L,_.matcher.considerAll()),y=_.matcher.exec(s);)O=b(s.substring(L,y.index),y),L=y.index+O;return b(s.substr(L)),w.closeAllNodes(),w.finalize(),x=w.toHTML(),{relevance:T,value:x,language:e,illegal:!1,emitter:w,top:_}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:s.slice(L-100,L+100),mode:n.mode},sofar:x,relevance:0,value:N(s),emitter:w};if(o)return{relevance:0,value:N(s),emitter:w,language:e,top:_,errorRaised:n};throw n}}function m(e,n){n=n||g.languages||Object.keys(a);var t=function(e){const n={relevance:0,emitter:new g.__emitter(g),value:N(e),illegal:!1,top:E};return n.emitter.addText(e),n}(e),r=t;return n.filter(M).filter(k).forEach((function(n){var a=p(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function b(e){return g.tabReplace||g.useBR?e.replace(l,(function(e,n){return g.useBR&&"\n"===e?"
":g.tabReplace?n.replace(/\t/g,g.tabReplace):""})):e}function v(e){var n,t,r,a,s,o=function(e){var n,t=e.className+" ";if(t+=e.parentNode?e.parentNode.className:"",n=g.languageDetectRe.exec(t)){var r=M(n[1]);return r||(console.warn(d.replace("{}",n[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?n[1]:"no-highlight"}return t.split(/\s+/).find(e=>h(e)||M(e))}(e);h(o)||(T("before:highlightBlock",{block:e,language:o}),g.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e,s=n.textContent,r=o?f(o,s,!0):m(s),(t=y(n)).length&&((a=document.createElement("div")).innerHTML=r.value,r.value=O(t,y(a),s)),r.value=b(r.value),T("after:highlightBlock",{block:e,result:r}),e.innerHTML=r.value,e.className=function(e,n,t){var r=n?i[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,o,r.language),e.result={language:r.language,re:r.relevance},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.relevance}))}function x(){if(!x.called){x.called=!0;var e=document.querySelectorAll("pre code");r.forEach.call(e,v)}}const E={disableAutodetect:!0,name:"Plain text"};function M(e){return e=(e||"").toLowerCase(),a[e]||a[i[e]]}function k(e){var n=M(e);return n&&!n.disableAutodetect}function T(e,n){var t=e;s.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(n,{highlight:f,highlightAuto:m,fixMarkup:b,highlightBlock:v,configure:function(e){g=w(g,e)},initHighlighting:x,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",x,!1)},registerLanguage:function(e,t){var r;try{r=t(n)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!o)throw n;console.error(n),r=E}r.name||(r.name=e),a[e]=r,r.rawDefinition=t.bind(null,n),r.aliases&&r.aliases.forEach((function(n){i[n]=e}))},listLanguages:function(){return Object.keys(a)},getLanguage:M,requireLanguage:function(e){var n=M(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:k,inherit:w,addPlugin:function(e,n){s.push(e)}}),n.debugMode=function(){o=!1},n.safeMode=function(){o=!0},n.versionString="10.0.3";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(n,_),n}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},{begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\((?:.|\n)*?\)\1"/}]},s={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},i={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},o=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+o,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:o,returnBegin:!0,contains:[c],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,s,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:i,strings:a,keywords:l}}}}());hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}());hljs.registerLanguage("xml",function(){"use strict";return function(e){var n={className:"symbol",begin:"&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;"},a={begin:"\\s",contains:[{className:"meta-keyword",begin:"#?[a-z_][a-z1-9_-]+",illegal:"\\n"}]},s=e.inherit(a,{begin:"\\(",end:"\\)"}),t=e.inherit(e.APOS_STRING_MODE,{className:"meta-string"}),i=e.inherit(e.QUOTE_STRING_MODE,{className:"meta-string"}),c={endsWithParent:!0,illegal:/`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}());hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}());hljs.registerLanguage("coffeescript",function(){"use strict";return function(e){var n={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger super yield import export from as default await then unless until loop of by when and or is isnt not",literal:"true false null undefined yes no on off",built_in:"npm require console print module global window document"},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:n},a=[e.BINARY_NUMBER_MODE,e.inherit(e.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[e.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,e.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=a;var t=e.inherit(e.TITLE_MODE,{begin:i}),r={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:n,contains:["self"].concat(a)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:n,illegal:/\/\*/,contains:a.concat([e.COMMENT("###","###"),e.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[t,r]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[r]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[t]},t]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}());hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const n={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},t={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,n]};n.contains.push(t);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]};return{name:"Bash",aliases:["sh","zsh"],lexemes:/\b-?[a-z\._]+\b/,keywords:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[{className:"meta",begin:/^#![^\n]+sh\s*$/,relevance:10},{className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0},a,e.HASH_COMMENT_MODE,t,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}());hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}());hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}());hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}());hljs.registerLanguage("javascript",function(){"use strict";return function(e){var n={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},a="[A-Za-z$_][0-9A-Za-z$_]*",s={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},r={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:e.C_NUMBER_RE+"n?"}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:s,contains:[]},t={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"xml"}},c={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"css"}},o={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,r,e.REGEXP_MODE];var l=i.contains.concat([e.C_BLOCK_COMMENT_MODE,e.C_LINE_COMMENT_MODE]),d={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:l};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:s,contains:[{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},{className:"meta",begin:/^#!/,end:/$/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,t,c,o,e.C_LINE_COMMENT_MODE,e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:a+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),e.C_BLOCK_COMMENT_MODE,r,{begin:/[{,\n]\s*/,relevance:0,contains:[{begin:a+"\\s*:",returnBegin:!0,relevance:0,contains:[{className:"attr",begin:a,relevance:0}]}]},{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+a+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:a},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:s,contains:l}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:n.begin,end:n.end}],subLanguage:"xml",contains:[{begin:n.begin,end:n.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:a}),d],illegal:/\[|%/},{begin:/\$[(.]/},e.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+a+"\\()",end:/{/,keywords:"get set",contains:[e.inherit(e.TITLE_MODE,{begin:a}),{begin:/\(\)/},d]}],illegal:/#(?!!)/}}}());hljs.registerLanguage("typescript",function(){"use strict";return function(e){var n={keyword:"in if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const class public private protected get set super static implements enum export import declare type namespace abstract as from extends async await",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document any number boolean string void Promise"},r={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},a={begin:"\\(",end:/\)/,keywords:n,contains:["self",e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.NUMBER_MODE]},t={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,r,a]},s={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:e.C_NUMBER_RE+"n?"}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:n,contains:[]},o={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"xml"}},c={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[e.BACKSLASH_ESCAPE,i],subLanguage:"css"}},E={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};return i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,c,E,s,e.REGEXP_MODE],{name:"TypeScript",aliases:["ts"],keywords:n,contains:[{className:"meta",begin:/^\s*['"]use strict['"]/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,c,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+e.IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:e.IDENT_RE},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]}]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/[\{;]/,excludeEnd:!0,keywords:n,contains:["self",e.inherit(e.TITLE_MODE,{begin:"[A-Za-z$_][0-9A-Za-z$_]*"}),t],illegal:/%/,relevance:0},{beginKeywords:"constructor",end:/[\{;]/,excludeEnd:!0,contains:["self",t]},{begin:/module\./,keywords:{built_in:"module"},relevance:0},{beginKeywords:"module",end:/\{/,excludeEnd:!0},{beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:"interface extends"},{begin:/\$[(.]/},{begin:"\\."+e.IDENT_RE,relevance:0},r,a]}}}());hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}());hljs.registerLanguage("http",function(){"use strict";return function(e){var n="HTTP/[0-9\\.]+";return{name:"HTTP",aliases:["https"],illegal:"\\S",contains:[{begin:"^"+n,end:"$",contains:[{className:"number",begin:"\\b\\d{3}\\b"}]},{begin:"^[A-Z]+ (.*?) "+n+"$",returnBegin:!0,end:"$",contains:[{className:"string",begin:" ",end:" ",excludeBegin:!0,excludeEnd:!0},{begin:n},{className:"keyword",begin:"[A-Z]+"}]},{className:"attribute",begin:"^\\w",end:": ",excludeEnd:!0,illegal:"\\n|\\s|=",starts:{end:"$",relevance:0}},{begin:"\\n\\n",starts:{subLanguage:[],endsWithParent:!0}}]}}}()); \ No newline at end of file diff --git a/doc/template.html b/doc/template.html index 2f330a41718af2..6caf5a82bc31a7 100644 --- a/doc/template.html +++ b/doc/template.html @@ -3,6 +3,7 @@ + __SECTION__ | Node.js __VERSION__ Documentation @@ -48,10 +49,9 @@

Table of Contents

__CONTENT__ +
- - diff --git a/tools/doc/allhtml.js b/tools/doc/allhtml.js index 3de538e42404a5..6d18d1be3f8f1f 100644 --- a/tools/doc/allhtml.js +++ b/tools/doc/allhtml.js @@ -37,7 +37,7 @@ for (const link of toc.match(//g)) { .replace(/[\s\S]*?
\s*

.*?<\/h2>\s*(

 - + diff --git a/doc/api/repl.md b/doc/api/repl.md index b58c1fb192085b..3ab7cefec280d2 100644 --- a/doc/api/repl.md +++ b/doc/api/repl.md @@ -246,8 +246,8 @@ added: v12.17.0 --> The REPL supports bi-directional reverse-i-search similar to [ZSH][]. It is -triggered with ` + R` to search backwards and ` + S` to search -forwards. +triggered with ` + R` to search backward and ` + S` to search +forward. Duplicated history entires will be skipped. diff --git a/doc/guides/technical-values.md b/doc/guides/technical-values.md index a688c0b21fe412..d79fde6a461021 100644 --- a/doc/guides/technical-values.md +++ b/doc/guides/technical-values.md @@ -26,7 +26,7 @@ with Node.js. Some key elements of this include: Whenever possible, we seek to ensure that working code continues to work. To keep the trust of developers and users, we value stability. Some key elements of this include: -* Backwards compatibility +* Backward compatibility * Stable releases on a predictable schedule * A strong safety net, including testing how changes in Node.js affect popular packages From 1d3638b189eb8f28c590b3f46a0a59ce7fd429e0 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 17 May 2020 07:13:16 +0200 Subject: [PATCH 398/492] src: use enum for refed flag on native immediates Refs: https://github.com/nodejs/node/pull/33320#discussion_r423141443 PR-URL: https://github.com/nodejs/node/pull/33444 Backport-PR-URL: https://github.com/nodejs/node/pull/34263 Reviewed-By: James M Snell Reviewed-By: Zeyu Yang Reviewed-By: Colin Ihrig Reviewed-By: David Carlier Reviewed-By: Ben Noordhuis --- src/async_wrap.cc | 2 +- src/callback_queue-inl.h | 16 ++++++++-------- src/callback_queue.h | 18 +++++++++++++----- src/env-inl.h | 32 ++++++++++++-------------------- src/env.cc | 7 ++++--- src/env.h | 11 ++++------- src/node_dir.cc | 4 ++-- src/node_file.cc | 5 +++-- src/node_perf.cc | 4 ++-- src/node_worker.cc | 2 +- test/cctest/test_environment.cc | 4 ++-- 11 files changed, 52 insertions(+), 53 deletions(-) diff --git a/src/async_wrap.cc b/src/async_wrap.cc index f260afa0dd5cc4..2ffe38580bc281 100644 --- a/src/async_wrap.cc +++ b/src/async_wrap.cc @@ -685,7 +685,7 @@ void AsyncWrap::EmitDestroy(Environment* env, double async_id) { } if (env->destroy_async_id_list()->empty()) { - env->SetUnrefImmediate(&DestroyAsyncIdsCallback); + env->SetImmediate(&DestroyAsyncIdsCallback, CallbackFlags::kUnrefed); } // If the list gets very large empty it faster using a Microtask. diff --git a/src/callback_queue-inl.h b/src/callback_queue-inl.h index 8c6919912e161b..9e46ae48699320 100644 --- a/src/callback_queue-inl.h +++ b/src/callback_queue-inl.h @@ -10,8 +10,8 @@ namespace node { template template std::unique_ptr::Callback> -CallbackQueue::CreateCallback(Fn&& fn, bool refed) { - return std::make_unique>(std::move(fn), refed); +CallbackQueue::CreateCallback(Fn&& fn, CallbackFlags::Flags flags) { + return std::make_unique>(std::move(fn), flags); } template @@ -57,12 +57,12 @@ size_t CallbackQueue::size() const { } template -CallbackQueue::Callback::Callback(bool refed) - : refed_(refed) {} +CallbackQueue::Callback::Callback(CallbackFlags::Flags flags) + : flags_(flags) {} template -bool CallbackQueue::Callback::is_refed() const { - return refed_; +CallbackFlags::Flags CallbackQueue::Callback::flags() const { + return flags_; } template @@ -80,8 +80,8 @@ void CallbackQueue::Callback::set_next( template template CallbackQueue::CallbackImpl::CallbackImpl( - Fn&& callback, bool refed) - : Callback(refed), + Fn&& callback, CallbackFlags::Flags flags) + : Callback(flags), callback_(std::move(callback)) {} template diff --git a/src/callback_queue.h b/src/callback_queue.h index ebf975e6391d13..e5694d5e1fe56a 100644 --- a/src/callback_queue.h +++ b/src/callback_queue.h @@ -7,6 +7,13 @@ namespace node { +namespace CallbackFlags { +enum Flags { + kUnrefed = 0, + kRefed = 1, +}; +} + // A queue of C++ functions that take Args... as arguments and return R // (this is similar to the signature of std::function). // New entries are added using `CreateCallback()`/`Push()`, and removed using @@ -18,25 +25,26 @@ class CallbackQueue { public: class Callback { public: - explicit inline Callback(bool refed); + explicit inline Callback(CallbackFlags::Flags flags); virtual ~Callback() = default; virtual R Call(Args... args) = 0; - inline bool is_refed() const; + inline CallbackFlags::Flags flags() const; private: inline std::unique_ptr get_next(); inline void set_next(std::unique_ptr next); - bool refed_; + CallbackFlags::Flags flags_; std::unique_ptr next_; friend class CallbackQueue; }; template - inline std::unique_ptr CreateCallback(Fn&& fn, bool refed); + inline std::unique_ptr CreateCallback( + Fn&& fn, CallbackFlags::Flags); inline std::unique_ptr Shift(); inline void Push(std::unique_ptr cb); @@ -51,7 +59,7 @@ class CallbackQueue { template class CallbackImpl final : public Callback { public: - CallbackImpl(Fn&& callback, bool refed); + CallbackImpl(Fn&& callback, CallbackFlags::Flags flags); R Call(Args... args) override; private: diff --git a/src/env-inl.h b/src/env-inl.h index 2acbe464ccaac4..acb939ce72b56c 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -786,29 +786,21 @@ inline void IsolateData::set_options( } template -void Environment::CreateImmediate(Fn&& cb, bool ref) { - auto callback = native_immediates_.CreateCallback(std::move(cb), ref); +void Environment::SetImmediate(Fn&& cb, CallbackFlags::Flags flags) { + auto callback = native_immediates_.CreateCallback(std::move(cb), flags); native_immediates_.Push(std::move(callback)); -} - -template -void Environment::SetImmediate(Fn&& cb) { - CreateImmediate(std::move(cb), true); - if (immediate_info()->ref_count() == 0) - ToggleImmediateRef(true); - immediate_info()->ref_count_inc(1); -} - -template -void Environment::SetUnrefImmediate(Fn&& cb) { - CreateImmediate(std::move(cb), false); + if (flags & CallbackFlags::kRefed) { + if (immediate_info()->ref_count() == 0) + ToggleImmediateRef(true); + immediate_info()->ref_count_inc(1); + } } template -void Environment::SetImmediateThreadsafe(Fn&& cb, bool refed) { - auto callback = - native_immediates_threadsafe_.CreateCallback(std::move(cb), refed); +void Environment::SetImmediateThreadsafe(Fn&& cb, CallbackFlags::Flags flags) { + auto callback = native_immediates_threadsafe_.CreateCallback( + std::move(cb), flags); { Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); native_immediates_threadsafe_.Push(std::move(callback)); @@ -818,8 +810,8 @@ void Environment::SetImmediateThreadsafe(Fn&& cb, bool refed) { template void Environment::RequestInterrupt(Fn&& cb) { - auto callback = - native_immediates_interrupts_.CreateCallback(std::move(cb), false); + auto callback = native_immediates_interrupts_.CreateCallback( + std::move(cb), CallbackFlags::kRefed); { Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); native_immediates_interrupts_.Push(std::move(callback)); diff --git a/src/env.cc b/src/env.cc index 52618f2033b76c..a0d98637a27b79 100644 --- a/src/env.cc +++ b/src/env.cc @@ -542,7 +542,7 @@ void Environment::CleanupHandles() { Isolate::DisallowJavascriptExecutionScope disallow_js(isolate(), Isolate::DisallowJavascriptExecutionScope::THROW_ON_FAILURE); - RunAndClearNativeImmediates(true /* skip SetUnrefImmediate()s */); + RunAndClearNativeImmediates(true /* skip unrefed SetImmediate()s */); for (ReqWrapBase* request : req_wrap_queue_) request->Cancel(); @@ -675,10 +675,11 @@ void Environment::RunAndClearNativeImmediates(bool only_refed) { TryCatchScope try_catch(this); DebugSealHandleScope seal_handle_scope(isolate()); while (auto head = queue->Shift()) { - if (head->is_refed()) + bool is_refed = head->flags() & CallbackFlags::kRefed; + if (is_refed) ref_count++; - if (head->is_refed() || !only_refed) + if (is_refed || !only_refed) head->Call(this); head.reset(); // Destroy now so that this is also observed by try_catch. diff --git a/src/env.h b/src/env.h index 85f3bd99ac92e9..0632a96260f4d6 100644 --- a/src/env.h +++ b/src/env.h @@ -1192,12 +1192,12 @@ class Environment : public MemoryRetainer { // Unlike the JS setImmediate() function, nested SetImmediate() calls will // be run without returning control to the event loop, similar to nextTick(). template - inline void SetImmediate(Fn&& cb); - template - inline void SetUnrefImmediate(Fn&& cb); + inline void SetImmediate( + Fn&& cb, CallbackFlags::Flags flags = CallbackFlags::kRefed); template // This behaves like SetImmediate() but can be called from any thread. - inline void SetImmediateThreadsafe(Fn&& cb, bool refed = true); + inline void SetImmediateThreadsafe( + Fn&& cb, CallbackFlags::Flags flags = CallbackFlags::kRefed); // This behaves like V8's Isolate::RequestInterrupt(), but also accounts for // the event loop (i.e. combines the V8 function with SetImmediate()). // The passed callback may not throw exceptions. @@ -1277,9 +1277,6 @@ class Environment : public MemoryRetainer { std::shared_ptr); private: - template - inline void CreateImmediate(Fn&& cb, bool ref); - inline void ThrowError(v8::Local (*fun)(v8::Local), const char* errmsg); diff --git a/src/node_dir.cc b/src/node_dir.cc index 6cda8de7ce648b..f4d2d1efbdc6e7 100644 --- a/src/node_dir.cc +++ b/src/node_dir.cc @@ -123,10 +123,10 @@ inline void DirHandle::GCClose() { // to notify that the file descriptor was gc'd. We want to be noisy about // this because not explicitly closing the DirHandle is a bug. - env()->SetUnrefImmediate([](Environment* env) { + env()->SetImmediate([](Environment* env) { ProcessEmitWarning(env, "Closing directory handle on garbage collection"); - }); + }, CallbackFlags::kUnrefed); } void AfterClose(uv_fs_t* req) { diff --git a/src/node_file.cc b/src/node_file.cc index 3ec72120d66201..0cce86e8f36dc3 100644 --- a/src/node_file.cc +++ b/src/node_file.cc @@ -221,11 +221,12 @@ inline void FileHandle::Close() { // If the close was successful, we still want to emit a process warning // to notify that the file descriptor was gc'd. We want to be noisy about // this because not explicitly closing the FileHandle is a bug. - env()->SetUnrefImmediate([detail](Environment* env) { + + env()->SetImmediate([detail](Environment* env) { ProcessEmitWarning(env, "Closing file descriptor %d on garbage collection", detail.fd); - }); + }, CallbackFlags::kUnrefed); } void FileHandle::CloseReq::Resolve() { diff --git a/src/node_perf.cc b/src/node_perf.cc index b3d64cf5755539..4a1eaac2fcf10d 100644 --- a/src/node_perf.cc +++ b/src/node_perf.cc @@ -279,9 +279,9 @@ void MarkGarbageCollectionEnd(Isolate* isolate, static_cast(flags), state->performance_last_gc_start_mark, PERFORMANCE_NOW()); - env->SetUnrefImmediate([entry = std::move(entry)](Environment* env) mutable { + env->SetImmediate([entry = std::move(entry)](Environment* env) mutable { PerformanceGCCallback(env, std::move(entry)); - }); + }, CallbackFlags::kUnrefed); } void GarbageCollectionCleanupHook(void* data) { diff --git a/src/node_worker.cc b/src/node_worker.cc index 5988c55e7b92b8..440f09f7b66e96 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -762,7 +762,7 @@ void Worker::TakeHeapSnapshot(const FunctionCallbackInfo& args) { env, std::move(snapshot)); Local args[] = { stream->object() }; taker->MakeCallback(env->ondone_string(), arraysize(args), args); - }, /* refed */ false); + }, CallbackFlags::kUnrefed); }); args.GetReturnValue().Set(scheduled ? taker->object() : Local()); } diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index a15e56a772ed19..75aa4b7d840e12 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -232,10 +232,10 @@ TEST_F(EnvironmentTest, SetImmediateCleanup) { EXPECT_EQ(env_arg, *env); called++; }); - (*env)->SetUnrefImmediate([&](node::Environment* env_arg) { + (*env)->SetImmediate([&](node::Environment* env_arg) { EXPECT_EQ(env_arg, *env); called_unref++; - }); + }, node::CallbackFlags::kUnrefed); } EXPECT_EQ(called, 1); From 7d92ac7a3584cd11690e2f1d1745f4b5b02e1a17 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 10 Nov 2019 19:47:03 +0000 Subject: [PATCH 399/492] src: make `FreeEnvironment()` perform all necessary cleanup Make the calls `stop_sub_worker_contexts()`, `RunCleanup()` part of the public API for easier embedding. (Note that calling `RunAtExit()` is idempotent because the at-exit callback queue is cleared after each call.) Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 18 +++++++++++++++++- src/node_main_instance.cc | 18 ++++++------------ src/node_main_instance.h | 3 ++- src/node_platform.cc | 5 ++++- src/node_worker.cc | 26 +++++++++----------------- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 97d25cba579570..d4fa856b8d59ea 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -342,7 +342,23 @@ Environment* CreateEnvironment(IsolateData* isolate_data, } void FreeEnvironment(Environment* env) { - env->RunCleanup(); + { + // TODO(addaleax): This should maybe rather be in a SealHandleScope. + HandleScope handle_scope(env->isolate()); + Context::Scope context_scope(env->context()); + env->set_stopping(true); + env->stop_sub_worker_contexts(); + env->RunCleanup(); + RunAtExit(env); + } + + // This call needs to be made while the `Environment` is still alive + // because we assume that it is available for async tracking in the + // NodePlatform implementation. + MultiIsolatePlatform* platform = env->isolate_data()->platform(); + if (platform != nullptr) + platform->DrainTasks(env->isolate()); + delete env; } diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 91bb30cb4e3ee6..c6533321bd6b64 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -110,7 +110,8 @@ int NodeMainInstance::Run() { HandleScope handle_scope(isolate_); int exit_code = 0; - std::unique_ptr env = CreateMainEnvironment(&exit_code); + DeleteFnPtr env = + CreateMainEnvironment(&exit_code); CHECK_NOT_NULL(env); Context::Scope context_scope(env->context()); @@ -149,10 +150,7 @@ int NodeMainInstance::Run() { exit_code = EmitExit(env.get()); } - env->set_can_call_into_js(false); - env->stop_sub_worker_contexts(); ResetStdio(); - env->RunCleanup(); // TODO(addaleax): Neither NODE_SHARED_MODE nor HAVE_INSPECTOR really // make sense here. @@ -167,10 +165,6 @@ int NodeMainInstance::Run() { } #endif - RunAtExit(env.get()); - - per_process::v8_platform.DrainVMTasks(isolate_); - #if defined(LEAK_SANITIZER) __lsan_do_leak_check(); #endif @@ -180,8 +174,8 @@ int NodeMainInstance::Run() { // TODO(joyeecheung): align this with the CreateEnvironment exposed in node.h // and the environment creation routine in workers somehow. -std::unique_ptr NodeMainInstance::CreateMainEnvironment( - int* exit_code) { +DeleteFnPtr +NodeMainInstance::CreateMainEnvironment(int* exit_code) { *exit_code = 0; // Reset the exit code to 0 HandleScope handle_scope(isolate_); @@ -205,14 +199,14 @@ std::unique_ptr NodeMainInstance::CreateMainEnvironment( CHECK(!context.IsEmpty()); Context::Scope context_scope(context); - std::unique_ptr env = std::make_unique( + DeleteFnPtr env { new Environment( isolate_data_.get(), context, args_, exec_args_, static_cast(Environment::kIsMainThread | Environment::kOwnsProcessState | - Environment::kOwnsInspector)); + Environment::kOwnsInspector)) }; env->InitializeLibuv(per_process::v8_is_profiling); env->InitializeDiagnostics(); diff --git a/src/node_main_instance.h b/src/node_main_instance.h index 5bc18cb3de63c0..cc9f50b9222de3 100644 --- a/src/node_main_instance.h +++ b/src/node_main_instance.h @@ -61,7 +61,8 @@ class NodeMainInstance { // TODO(joyeecheung): align this with the CreateEnvironment exposed in node.h // and the environment creation routine in workers somehow. - std::unique_ptr CreateMainEnvironment(int* exit_code); + DeleteFnPtr CreateMainEnvironment( + int* exit_code); // If nullptr is returned, the binary is not built with embedded // snapshot. diff --git a/src/node_platform.cc b/src/node_platform.cc index 5b878f886a1320..e6e880b87b5380 100644 --- a/src/node_platform.cc +++ b/src/node_platform.cc @@ -402,6 +402,7 @@ void PerIsolatePlatformData::RunForegroundTask(uv_timer_t* handle) { void NodePlatform::DrainTasks(Isolate* isolate) { std::shared_ptr per_isolate = ForIsolate(isolate); + if (!per_isolate) return; do { // Worker tasks aren't associated with an Isolate. @@ -468,7 +469,9 @@ NodePlatform::ForIsolate(Isolate* isolate) { } bool NodePlatform::FlushForegroundTasks(Isolate* isolate) { - return ForIsolate(isolate)->FlushForegroundTasksInternal(); + std::shared_ptr per_isolate = ForIsolate(isolate); + if (!per_isolate) return false; + return per_isolate->FlushForegroundTasksInternal(); } bool NodePlatform::IdleTasksEnabled(Isolate* isolate) { return false; } diff --git a/src/node_worker.cc b/src/node_worker.cc index 440f09f7b66e96..ec0fdf1cc0fad1 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -280,26 +280,18 @@ void Worker::Run() { if (!env_) return; env_->set_can_call_into_js(false); - Isolate::DisallowJavascriptExecutionScope disallow_js(isolate_, - Isolate::DisallowJavascriptExecutionScope::THROW_ON_FAILURE); { - Context::Scope context_scope(env_->context()); - { - Mutex::ScopedLock lock(mutex_); - stopped_ = true; - this->env_ = nullptr; - } - env_->set_stopping(true); - env_->stop_sub_worker_contexts(); - env_->RunCleanup(); - RunAtExit(env_.get()); - - // This call needs to be made while the `Environment` is still alive - // because we assume that it is available for async tracking in the - // NodePlatform implementation. - platform_->DrainTasks(isolate_); + Mutex::ScopedLock lock(mutex_); + stopped_ = true; + this->env_ = nullptr; } + + // TODO(addaleax): Try moving DisallowJavascriptExecutionScope into + // FreeEnvironment(). + Isolate::DisallowJavascriptExecutionScope disallow_js(isolate_, + Isolate::DisallowJavascriptExecutionScope::THROW_ON_FAILURE); + env_.reset(); }); if (is_stopped()) return; From 9a5cec3466060246ea1b9708371ddc039d60571b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 11 Nov 2019 12:29:07 +0000 Subject: [PATCH 400/492] src: fix memory leak in CreateEnvironment when bootstrap fails Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index d4fa856b8d59ea..ad496a6621d10c 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -336,8 +336,10 @@ Environment* CreateEnvironment(IsolateData* isolate_data, Environment::kOwnsProcessState | Environment::kOwnsInspector)); env->InitializeLibuv(per_process::v8_is_profiling); - if (env->RunBootstrapping().IsEmpty()) + if (env->RunBootstrapping().IsEmpty()) { + FreeEnvironment(env); return nullptr; + } return env; } From b7350e8c6e145812fcbb74c19f6826f06e261a0e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 12 Nov 2019 18:57:10 +0000 Subject: [PATCH 401/492] src: move worker_context from Environment to IsolateData Workers are fully in control of their Isolates, and this helps avoid a problem with later changes to `CreateEnvironment()` because now we can run the boostrapping code inside the latter. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/env-inl.h | 16 ++++++++++------ src/env.cc | 6 +++--- src/env.h | 7 ++++--- src/node_worker.cc | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/env-inl.h b/src/env-inl.h index acb939ce72b56c..93b71c3194fa56 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -69,6 +69,15 @@ inline v8::Local IsolateData::async_wrap_provider(int index) const { return async_wrap_providers_[index].Get(isolate_); } +inline void IsolateData::set_worker_context(worker::Worker* context) { + CHECK_NULL(worker_context_); // Should be set only once. + worker_context_ = context; +} + +inline worker::Worker* IsolateData::worker_context() const { + return worker_context_; +} + inline AsyncHooks::AsyncHooks() : async_ids_stack_(env()->isolate(), 16 * 2), fields_(env()->isolate(), kFieldsCount), @@ -861,12 +870,7 @@ inline uint64_t Environment::thread_id() const { } inline worker::Worker* Environment::worker_context() const { - return worker_context_; -} - -inline void Environment::set_worker_context(worker::Worker* context) { - CHECK_NULL(worker_context_); // Should be set only once. - worker_context_ = context; + return isolate_data()->worker_context(); } inline void Environment::add_sub_worker_context(worker::Worker* context) { diff --git a/src/env.cc b/src/env.cc index a0d98637a27b79..653f2e3f9e8753 100644 --- a/src/env.cc +++ b/src/env.cc @@ -973,7 +973,7 @@ void Environment::Exit(int exit_code) { DisposePlatform(); exit(exit_code); } else { - worker_context_->Exit(exit_code); + worker_context()->Exit(exit_code); } } @@ -987,8 +987,8 @@ void Environment::stop_sub_worker_contexts() { } Environment* Environment::worker_parent_env() const { - if (worker_context_ == nullptr) return nullptr; - return worker_context_->env(); + if (worker_context() == nullptr) return nullptr; + return worker_context()->env(); } void Environment::BuildEmbedderGraph(Isolate* isolate, diff --git a/src/env.h b/src/env.h index 0632a96260f4d6..72381c6cf4ad4e 100644 --- a/src/env.h +++ b/src/env.h @@ -504,6 +504,9 @@ class IsolateData : public MemoryRetainer { inline v8::ArrayBuffer::Allocator* allocator() const; inline NodeArrayBufferAllocator* node_allocator() const; + inline worker::Worker* worker_context() const; + inline void set_worker_context(worker::Worker* context); + #define VP(PropertyName, StringValue) V(v8::Private, PropertyName) #define VY(PropertyName, StringValue) V(v8::Symbol, PropertyName) #define VS(PropertyName, StringValue) V(v8::String, PropertyName) @@ -552,6 +555,7 @@ class IsolateData : public MemoryRetainer { const bool uses_node_allocator_; MultiIsolatePlatform* platform_; std::shared_ptr options_; + worker::Worker* worker_context_ = nullptr; }; struct ContextInfo { @@ -1074,7 +1078,6 @@ class Environment : public MemoryRetainer { inline uint64_t thread_id() const; inline worker::Worker* worker_context() const; Environment* worker_parent_env() const; - inline void set_worker_context(worker::Worker* context); inline void add_sub_worker_context(worker::Worker* context); inline void remove_sub_worker_context(worker::Worker* context); void stop_sub_worker_contexts(); @@ -1390,8 +1393,6 @@ class Environment : public MemoryRetainer { std::vector> file_handle_read_wrap_freelist_; - worker::Worker* worker_context_ = nullptr; - std::list extra_linked_bindings_; Mutex extra_linked_bindings_mutex_; diff --git a/src/node_worker.cc b/src/node_worker.cc index ec0fdf1cc0fad1..5cac7d0a975de5 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -185,6 +185,7 @@ class WorkerThreadData { CHECK(isolate_data_); if (w_->per_isolate_opts_) isolate_data_->set_options(std::move(w_->per_isolate_opts_)); + isolate_data_->set_worker_context(w_); } Mutex::ScopedLock lock(w_->mutex_); @@ -327,7 +328,6 @@ void Worker::Run() { CHECK_NOT_NULL(env_); env_->set_env_vars(std::move(env_vars_)); env_->set_abort_on_uncaught_exception(false); - env_->set_worker_context(this); env_->InitializeLibuv(start_profiler_idle_notifier_); } From e809a5cd6b6a5e389b1d11e267ae74474257ac71 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 12 Nov 2019 19:36:07 +0000 Subject: [PATCH 402/492] src: associate is_main_thread() with worker_context() In our codebase, the assumption generally is that `!is_main_thread()` means that the current Environment belongs to a Node.js Worker thread. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 3 +-- src/env-inl.h | 2 +- src/env.h | 1 - src/node_main_instance.cc | 3 +-- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index ad496a6621d10c..ee7732c93b8563 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -332,8 +332,7 @@ Environment* CreateEnvironment(IsolateData* isolate_data, context, args, exec_args, - static_cast(Environment::kIsMainThread | - Environment::kOwnsProcessState | + static_cast(Environment::kOwnsProcessState | Environment::kOwnsInspector)); env->InitializeLibuv(per_process::v8_is_profiling); if (env->RunBootstrapping().IsEmpty()) { diff --git a/src/env-inl.h b/src/env-inl.h index 93b71c3194fa56..b82ac88fc7fafc 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -854,7 +854,7 @@ inline void Environment::set_has_serialized_options(bool value) { } inline bool Environment::is_main_thread() const { - return flags_ & kIsMainThread; + return worker_context() == nullptr; } inline bool Environment::owns_process_state() const { diff --git a/src/env.h b/src/env.h index 72381c6cf4ad4e..5e9c7c0495366f 100644 --- a/src/env.h +++ b/src/env.h @@ -876,7 +876,6 @@ class Environment : public MemoryRetainer { enum Flags { kNoFlags = 0, - kIsMainThread = 1 << 0, kOwnsProcessState = 1 << 1, kOwnsInspector = 1 << 2, }; diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index c6533321bd6b64..cbbff1ad742006 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -204,8 +204,7 @@ NodeMainInstance::CreateMainEnvironment(int* exit_code) { context, args_, exec_args_, - static_cast(Environment::kIsMainThread | - Environment::kOwnsProcessState | + static_cast(Environment::kOwnsProcessState | Environment::kOwnsInspector)) }; env->InitializeLibuv(per_process::v8_is_profiling); env->InitializeDiagnostics(); From 808dedc4b3230bcebb5c6272040d7bb6486d09c8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 11 Nov 2019 14:24:13 +0000 Subject: [PATCH 403/492] src: align worker and main thread code with embedder API This addresses some long-standing TODOs by Joyee and me about making the embedder API more powerful and us less reliant on internal APIs for creating the main thread and Workers. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 90 ++++++++++++++++++++++++++++++++++++--- src/env-inl.h | 11 +++-- src/env.cc | 47 ++++++++++++++------ src/env.h | 20 ++++----- src/node.cc | 41 +++++++++++++----- src/node.h | 52 +++++++++++++++++++++- src/node_internals.h | 1 + src/node_main_instance.cc | 16 ++----- src/node_main_instance.h | 2 - src/node_worker.cc | 81 +++++++++++++++-------------------- src/node_worker.h | 7 +-- 11 files changed, 253 insertions(+), 115 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index ee7732c93b8563..2f17e1b2c75100 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -7,6 +7,10 @@ #include "node_v8_platform-inl.h" #include "uv.h" +#if HAVE_INSPECTOR +#include "inspector/worker_inspector.h" // ParentInspectorHandle +#endif + namespace node { using errors::TryCatchScope; using v8::Array; @@ -319,26 +323,40 @@ Environment* CreateEnvironment(IsolateData* isolate_data, const char* const* argv, int exec_argc, const char* const* exec_argv) { + return CreateEnvironment( + isolate_data, context, + std::vector(argv, argv + argc), + std::vector(exec_argv, exec_argv + exec_argc)); +} + +Environment* CreateEnvironment( + IsolateData* isolate_data, + Local context, + const std::vector& args, + const std::vector& exec_args, + EnvironmentFlags::Flags flags, + ThreadId thread_id) { Isolate* isolate = context->GetIsolate(); HandleScope handle_scope(isolate); Context::Scope context_scope(context); // TODO(addaleax): This is a much better place for parsing per-Environment // options than the global parse call. - std::vector args(argv, argv + argc); - std::vector exec_args(exec_argv, exec_argv + exec_argc); - // TODO(addaleax): Provide more sensible flags, in an embedder-accessible way. Environment* env = new Environment( isolate_data, context, args, exec_args, - static_cast(Environment::kOwnsProcessState | - Environment::kOwnsInspector)); - env->InitializeLibuv(per_process::v8_is_profiling); + flags, + thread_id); + if (flags & EnvironmentFlags::kOwnsProcessState) { + env->set_abort_on_uncaught_exception(false); + } + if (env->RunBootstrapping().IsEmpty()) { FreeEnvironment(env); return nullptr; } + return env; } @@ -363,6 +381,58 @@ void FreeEnvironment(Environment* env) { delete env; } +InspectorParentHandle::~InspectorParentHandle() {} + +// Hide the internal handle class from the public API. +#if HAVE_INSPECTOR +struct InspectorParentHandleImpl : public InspectorParentHandle { + std::unique_ptr impl; + + explicit InspectorParentHandleImpl( + std::unique_ptr&& impl) + : impl(std::move(impl)) {} +}; +#endif + +NODE_EXTERN std::unique_ptr GetInspectorParentHandle( + Environment* env, + ThreadId thread_id, + const char* url) { + CHECK_NOT_NULL(env); + CHECK_NE(thread_id.id, static_cast(-1)); +#if HAVE_INSPECTOR + return std::make_unique( + env->inspector_agent()->GetParentHandle(thread_id.id, url)); +#else + return {}; +#endif +} + +void LoadEnvironment(Environment* env) { + USE(LoadEnvironment(env, {})); +} + +MaybeLocal LoadEnvironment( + Environment* env, + std::unique_ptr inspector_parent_handle) { + env->InitializeLibuv(per_process::v8_is_profiling); + env->InitializeDiagnostics(); + +#if HAVE_INSPECTOR + if (inspector_parent_handle) { + env->InitializeInspector( + std::move(static_cast( + inspector_parent_handle.get())->impl)); + } else { + env->InitializeInspector({}); + } +#endif + + // TODO(joyeecheung): Allow embedders to customize the entry + // point more directly without using _third_party_main.js + return StartExecution(env); +} + Environment* GetCurrentEnvironment(Local context) { return Environment::GetCurrent(context); } @@ -579,4 +649,12 @@ void AddLinkedBinding(Environment* env, AddLinkedBinding(env, mod); } +static std::atomic next_thread_id{0}; + +ThreadId AllocateEnvironmentThreadId() { + ThreadId ret; + ret.id = next_thread_id++; + return ret; +} + } // namespace node diff --git a/src/env-inl.h b/src/env-inl.h index b82ac88fc7fafc..598b84c07ca8d2 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -813,8 +813,9 @@ void Environment::SetImmediateThreadsafe(Fn&& cb, CallbackFlags::Flags flags) { { Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); native_immediates_threadsafe_.Push(std::move(callback)); + if (task_queues_async_initialized_) + uv_async_send(&task_queues_async_); } - uv_async_send(&task_queues_async_); } template @@ -824,8 +825,9 @@ void Environment::RequestInterrupt(Fn&& cb) { { Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); native_immediates_interrupts_.Push(std::move(callback)); + if (task_queues_async_initialized_) + uv_async_send(&task_queues_async_); } - uv_async_send(&task_queues_async_); RequestInterruptFromV8(); } @@ -858,11 +860,11 @@ inline bool Environment::is_main_thread() const { } inline bool Environment::owns_process_state() const { - return flags_ & kOwnsProcessState; + return flags_ & EnvironmentFlags::kOwnsProcessState; } inline bool Environment::owns_inspector() const { - return flags_ & kOwnsInspector; + return flags_ & EnvironmentFlags::kOwnsInspector; } inline uint64_t Environment::thread_id() const { @@ -1176,6 +1178,7 @@ void Environment::RemoveCleanupHook(void (*fn)(void*), void* arg) { inline void Environment::RegisterFinalizationGroupForCleanup( v8::Local group) { cleanup_finalization_groups_.emplace_back(isolate(), group); + DCHECK(task_queues_async_initialized_); uv_async_send(&task_queues_async_); } diff --git a/src/env.cc b/src/env.cc index 653f2e3f9e8753..f0b8b521f70363 100644 --- a/src/env.cc +++ b/src/env.cc @@ -257,12 +257,6 @@ void TrackingTraceStateObserver::UpdateTraceCategoryState() { USE(cb->Call(env_->context(), Undefined(isolate), arraysize(args), args)); } -static std::atomic next_thread_id{0}; - -uint64_t Environment::AllocateThreadId() { - return next_thread_id++; -} - void Environment::CreateProperties() { HandleScope handle_scope(isolate_); Local ctx = context(); @@ -319,8 +313,8 @@ Environment::Environment(IsolateData* isolate_data, Local context, const std::vector& args, const std::vector& exec_args, - Flags flags, - uint64_t thread_id) + EnvironmentFlags::Flags flags, + ThreadId thread_id) : isolate_(context->GetIsolate()), isolate_data_(isolate_data), immediate_info_(context->GetIsolate()), @@ -332,7 +326,8 @@ Environment::Environment(IsolateData* isolate_data, should_abort_on_uncaught_toggle_(isolate_, 1), stream_base_state_(isolate_, StreamBase::kNumStreamBaseStateFields), flags_(flags), - thread_id_(thread_id == kNoThreadId ? AllocateThreadId() : thread_id), + thread_id_(thread_id.id == static_cast(-1) ? + AllocateEnvironmentThreadId().id : thread_id.id), fs_stats_field_array_(isolate_, kFsStatsBufferLength), fs_stats_field_bigint_array_(isolate_, kFsStatsBufferLength), context_(context->GetIsolate(), context) { @@ -340,6 +335,14 @@ Environment::Environment(IsolateData* isolate_data, HandleScope handle_scope(isolate()); Context::Scope context_scope(context); + // Set some flags if only kDefaultFlags was passed. This can make API version + // transitions easier for embedders. + if (flags_ & EnvironmentFlags::kDefaultFlags) { + flags_ = flags_ | + EnvironmentFlags::kOwnsProcessState | + EnvironmentFlags::kOwnsInspector; + } + set_env_vars(per_process::system_environment); enabled_debug_list_.Parse(this); @@ -358,6 +361,10 @@ Environment::Environment(IsolateData* isolate_data, AssignToContext(context, ContextInfo("")); + static uv_once_t init_once = UV_ONCE_INIT; + uv_once(&init_once, InitThreadLocalOnce); + uv_key_set(&thread_local_env, this); + if (tracing::AgentWriterHandle* writer = GetTracingAgentWriter()) { trace_state_observer_ = std::make_unique(this); if (TracingController* tracing_controller = writer->GetTracingController()) @@ -407,6 +414,9 @@ Environment::Environment(IsolateData* isolate_data, Environment::~Environment() { if (interrupt_data_ != nullptr) *interrupt_data_ = nullptr; + // FreeEnvironment() should have set this. + CHECK(is_stopping()); + isolate()->GetHeapProfiler()->RemoveBuildEmbedderGraphCallback( BuildEmbedderGraph, this); @@ -493,6 +503,15 @@ void Environment::InitializeLibuv(bool start_profiler_idle_notifier) { uv_unref(reinterpret_cast(&idle_check_handle_)); uv_unref(reinterpret_cast(&task_queues_async_)); + { + Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); + task_queues_async_initialized_ = true; + if (native_immediates_threadsafe_.size() > 0 || + native_immediates_interrupts_.size() > 0) { + uv_async_send(&task_queues_async_); + } + } + // Register clean-up cb to be called to clean up the handles // when the environment is freed, note that they are not cleaned in // the one environment per process setup, but will be called in @@ -502,10 +521,6 @@ void Environment::InitializeLibuv(bool start_profiler_idle_notifier) { if (start_profiler_idle_notifier) { StartProfilerIdleNotifier(); } - - static uv_once_t init_once = UV_ONCE_INIT; - uv_once(&init_once, InitThreadLocalOnce); - uv_key_set(&thread_local_env, this); } void Environment::ExitEnv() { @@ -539,6 +554,11 @@ void Environment::RegisterHandleCleanups() { } void Environment::CleanupHandles() { + { + Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_); + task_queues_async_initialized_ = false; + } + Isolate::DisallowJavascriptExecutionScope disallow_js(isolate(), Isolate::DisallowJavascriptExecutionScope::THROW_ON_FAILURE); @@ -1103,6 +1123,7 @@ void Environment::CleanupFinalizationGroups() { if (try_catch.HasCaught() && !try_catch.HasTerminated()) errors::TriggerUncaughtException(isolate(), try_catch); // Re-schedule the execution of the remainder of the queue. + CHECK(task_queues_async_initialized_); uv_async_send(&task_queues_async_); return; } diff --git a/src/env.h b/src/env.h index 5e9c7c0495366f..8331a79dbc9918 100644 --- a/src/env.h +++ b/src/env.h @@ -874,12 +874,6 @@ class Environment : public MemoryRetainer { inline void PushAsyncCallbackScope(); inline void PopAsyncCallbackScope(); - enum Flags { - kNoFlags = 0, - kOwnsProcessState = 1 << 1, - kOwnsInspector = 1 << 2, - }; - static inline Environment* GetCurrent(v8::Isolate* isolate); static inline Environment* GetCurrent(v8::Local context); static inline Environment* GetCurrent( @@ -898,8 +892,8 @@ class Environment : public MemoryRetainer { v8::Local context, const std::vector& args, const std::vector& exec_args, - Flags flags = Flags(), - uint64_t thread_id = kNoThreadId); + EnvironmentFlags::Flags flags, + ThreadId thread_id); ~Environment() override; void InitializeLibuv(bool start_profiler_idle_notifier); @@ -1068,9 +1062,6 @@ class Environment : public MemoryRetainer { inline bool has_serialized_options() const; inline void set_has_serialized_options(bool has_serialized_options); - static uint64_t AllocateThreadId(); - static constexpr uint64_t kNoThreadId = -1; - inline bool is_main_thread() const; inline bool owns_process_state() const; inline bool owns_inspector() const; @@ -1350,7 +1341,7 @@ class Environment : public MemoryRetainer { bool has_serialized_options_ = false; std::atomic_bool can_call_into_js_ { true }; - Flags flags_; + uint64_t flags_; uint64_t thread_id_; std::unordered_set sub_worker_contexts_; @@ -1409,6 +1400,11 @@ class Environment : public MemoryRetainer { Mutex native_immediates_threadsafe_mutex_; NativeImmediateQueue native_immediates_threadsafe_; NativeImmediateQueue native_immediates_interrupts_; + // Also guarded by native_immediates_threadsafe_mutex_. This can be used when + // trying to post tasks from other threads to an Environment, as the libuv + // handle for the immediate queues (task_queues_async_) may not be initialized + // yet or already have been destroyed. + bool task_queues_async_initialized_ = false; void RunAndClearNativeImmediates(bool only_refed = false); void RunAndClearInterrupts(); diff --git a/src/node.cc b/src/node.cc index e93ddecb872f56..c34a875768315c 100644 --- a/src/node.cc +++ b/src/node.cc @@ -190,8 +190,8 @@ MaybeLocal ExecuteBootstrapper(Environment* env, int Environment::InitializeInspector( std::unique_ptr parent_handle) { std::string inspector_path; + bool is_main = !parent_handle; if (parent_handle) { - DCHECK(!is_main_thread()); inspector_path = parent_handle->url(); inspector_agent_->SetParentHandle(std::move(parent_handle)); } else { @@ -205,7 +205,7 @@ int Environment::InitializeInspector( inspector_agent_->Start(inspector_path, options_->debug_options(), inspector_host_port(), - is_main_thread()); + is_main); if (options_->debug_options().inspector_enabled && !inspector_agent_->IsListening()) { return 12; // Signal internal error @@ -394,7 +394,7 @@ MaybeLocal StartExecution(Environment* env, const char* main_script_id) { ExecuteBootstrapper(env, main_script_id, ¶meters, &arguments)); } -MaybeLocal StartMainThreadExecution(Environment* env) { +MaybeLocal StartExecution(Environment* env) { // To allow people to extend Node in different ways, this hook allows // one to drop a file lib/_third_party_main.js into the build // directory which will be executed instead of Node's normal loading. @@ -402,6 +402,10 @@ MaybeLocal StartMainThreadExecution(Environment* env) { return StartExecution(env, "internal/main/run_third_party_main"); } + if (env->worker_context() != nullptr) { + return StartExecution(env, "internal/main/worker_thread"); + } + std::string first_argv; if (env->argv().size() > 1) { first_argv = env->argv()[1]; @@ -440,15 +444,30 @@ MaybeLocal StartMainThreadExecution(Environment* env) { return StartExecution(env, "internal/main/eval_stdin"); } -void LoadEnvironment(Environment* env) { - CHECK(env->is_main_thread()); - // TODO(joyeecheung): Not all of the execution modes in - // StartMainThreadExecution() make sense for embedders. Pick the - // useful ones out, and allow embedders to customize the entry - // point more directly without using _third_party_main.js - USE(StartMainThreadExecution(env)); +#ifdef __POSIX__ +typedef void (*sigaction_cb)(int signo, siginfo_t* info, void* ucontext); +#endif +#if NODE_USE_V8_WASM_TRAP_HANDLER +static std::atomic previous_sigsegv_action; + +void TrapWebAssemblyOrContinue(int signo, siginfo_t* info, void* ucontext) { + if (!v8::TryHandleWebAssemblyTrapPosix(signo, info, ucontext)) { + sigaction_cb prev = previous_sigsegv_action.load(); + if (prev != nullptr) { + prev(signo, info, ucontext); + } else { + // Reset to the default signal handler, i.e. cause a hard crash. + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_DFL; + CHECK_EQ(sigaction(signo, &sa, nullptr), 0); + + ResetStdio(); + raise(signo); + } + } } - +#endif // NODE_USE_V8_WASM_TRAP_HANDLER #ifdef __POSIX__ void RegisterSignalHandler(int signal, diff --git a/src/node.h b/src/node.h index e9cf73ede5373a..4357291de79cec 100644 --- a/src/node.h +++ b/src/node.h @@ -363,18 +363,66 @@ NODE_EXTERN IsolateData* CreateIsolateData( ArrayBufferAllocator* allocator = nullptr); NODE_EXTERN void FreeIsolateData(IsolateData* isolate_data); -// TODO(addaleax): Add an official variant using STL containers, and move -// per-Environment options parsing here. +struct ThreadId { + uint64_t id = static_cast(-1); +}; +NODE_EXTERN ThreadId AllocateEnvironmentThreadId(); + +namespace EnvironmentFlags { +enum Flags : uint64_t { + kNoFlags = 0, + // Use the default behaviour for Node.js instances. + kDefaultFlags = 1 << 0, + // Controls whether this Environment is allowed to affect per-process state + // (e.g. cwd, process title, uid, etc.). + // This is set when using kDefaultFlags. + kOwnsProcessState = 1 << 1, + // Set if this Environment instance is associated with the global inspector + // handling code (i.e. listening on SIGUSR1). + // This is set when using kDefaultFlags. + kOwnsInspector = 1 << 2 +}; +} // namespace EnvironmentFlags + +// TODO(addaleax): Maybe move per-Environment options parsing here. // Returns nullptr when the Environment cannot be created e.g. there are // pending JavaScript exceptions. +// It is recommended to use the second variant taking a flags argument. NODE_EXTERN Environment* CreateEnvironment(IsolateData* isolate_data, v8::Local context, int argc, const char* const* argv, int exec_argc, const char* const* exec_argv); +NODE_EXTERN Environment* CreateEnvironment( + IsolateData* isolate_data, + v8::Local context, + const std::vector& args, + const std::vector& exec_args, + EnvironmentFlags::Flags flags = EnvironmentFlags::kDefaultFlags, + ThreadId thread_id = {} /* allocates a thread id automatically */); +struct InspectorParentHandle { + virtual ~InspectorParentHandle(); +}; +// Returns a handle that can be passed to `LoadEnvironment()`, making the +// child Environment accessible to the inspector as if it were a Node.js Worker. +// `child_thread_id` can be created using `AllocateEnvironmentThreadId()` +// and then later passed on to `CreateEnvironment()` to create the child +// Environment. +// This method should not be called while the parent Environment is active +// on another thread. +NODE_EXTERN std::unique_ptr GetInspectorParentHandle( + Environment* parent_env, + ThreadId child_thread_id, + const char* child_url); + +// TODO(addaleax): Deprecate this in favour of the MaybeLocal<> overload +// and provide a more flexible approach than third_party_main. NODE_EXTERN void LoadEnvironment(Environment* env); +NODE_EXTERN v8::MaybeLocal LoadEnvironment( + Environment* env, + std::unique_ptr inspector_parent_handle); NODE_EXTERN void FreeEnvironment(Environment* env); // This may return nullptr if context is not associated with a Node instance. diff --git a/src/node_internals.h b/src/node_internals.h index c1555b312e2f22..38de262e05e8d9 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -301,6 +301,7 @@ void DefineZlibConstants(v8::Local target); v8::Isolate* NewIsolate(v8::Isolate::CreateParams* params, uv_loop_t* event_loop, MultiIsolatePlatform* platform); +v8::MaybeLocal StartExecution(Environment* env); v8::MaybeLocal StartExecution(Environment* env, const char* main_script_id); v8::MaybeLocal GetPerContextExports(v8::Local context); diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index cbbff1ad742006..f638e26dba5a04 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -172,8 +172,6 @@ int NodeMainInstance::Run() { return exit_code; } -// TODO(joyeecheung): align this with the CreateEnvironment exposed in node.h -// and the environment creation routine in workers somehow. DeleteFnPtr NodeMainInstance::CreateMainEnvironment(int* exit_code) { *exit_code = 0; // Reset the exit code to 0 @@ -199,26 +197,18 @@ NodeMainInstance::CreateMainEnvironment(int* exit_code) { CHECK(!context.IsEmpty()); Context::Scope context_scope(context); - DeleteFnPtr env { new Environment( + DeleteFnPtr env { CreateEnvironment( isolate_data_.get(), context, args_, exec_args_, - static_cast(Environment::kOwnsProcessState | - Environment::kOwnsInspector)) }; - env->InitializeLibuv(per_process::v8_is_profiling); - env->InitializeDiagnostics(); + EnvironmentFlags::kDefaultFlags) }; - // TODO(joyeecheung): when we snapshot the bootstrapped context, - // the inspector and diagnostics setup should after after deserialization. -#if HAVE_INSPECTOR - *exit_code = env->InitializeInspector({}); -#endif if (*exit_code != 0) { return env; } - if (env->RunBootstrapping().IsEmpty()) { + if (env == nullptr) { *exit_code = 1; } diff --git a/src/node_main_instance.h b/src/node_main_instance.h index cc9f50b9222de3..b8178c2774e795 100644 --- a/src/node_main_instance.h +++ b/src/node_main_instance.h @@ -59,8 +59,6 @@ class NodeMainInstance { IsolateData* isolate_data() { return isolate_data_.get(); } - // TODO(joyeecheung): align this with the CreateEnvironment exposed in node.h - // and the environment creation routine in workers somehow. DeleteFnPtr CreateMainEnvironment( int* exit_code); diff --git a/src/node_worker.cc b/src/node_worker.cc index 5cac7d0a975de5..dcce464cf80f88 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -8,10 +8,6 @@ #include "util-inl.h" #include "async_wrap-inl.h" -#if HAVE_INSPECTOR -#include "inspector/worker_inspector.h" // ParentInspectorHandle -#endif - #include #include #include @@ -54,10 +50,10 @@ Worker::Worker(Environment* env, exec_argv_(exec_argv), platform_(env->isolate_data()->platform()), array_buffer_allocator_(ArrayBufferAllocator::Create()), - start_profiler_idle_notifier_(env->profiler_idle_notifier_started()), - thread_id_(Environment::AllocateThreadId()), + thread_id_(AllocateEnvironmentThreadId()), env_vars_(env_vars) { - Debug(this, "Creating new worker instance with thread id %llu", thread_id_); + Debug(this, "Creating new worker instance with thread id %llu", + thread_id_.id); // Set up everything that needs to be set up in the parent environment. parent_port_ = MessagePort::New(env, env->context()); @@ -75,19 +71,17 @@ Worker::Worker(Environment* env, object()->Set(env->context(), env->thread_id_string(), - Number::New(env->isolate(), static_cast(thread_id_))) + Number::New(env->isolate(), static_cast(thread_id_.id))) .Check(); -#if HAVE_INSPECTOR - inspector_parent_handle_ = - env->inspector_agent()->GetParentHandle(thread_id_, url); -#endif + inspector_parent_handle_ = GetInspectorParentHandle( + env, thread_id_, url.c_str()); argv_ = std::vector{env->argv()[0]}; // Mark this Worker object as weak until we actually start the thread. MakeWeak(); - Debug(this, "Preparation for worker %llu finished", thread_id_); + Debug(this, "Preparation for worker %llu finished", thread_id_.id); } bool Worker::is_stopped() const { @@ -193,7 +187,7 @@ class WorkerThreadData { } ~WorkerThreadData() { - Debug(w_, "Worker %llu dispose isolate", w_->thread_id_); + Debug(w_, "Worker %llu dispose isolate", w_->thread_id_.id); Isolate* isolate; { Mutex::ScopedLock lock(w_->mutex_); @@ -254,19 +248,19 @@ size_t Worker::NearHeapLimit(void* data, size_t current_heap_limit, void Worker::Run() { std::string name = "WorkerThread "; - name += std::to_string(thread_id_); + name += std::to_string(thread_id_.id); TRACE_EVENT_METADATA1( "__metadata", "thread_name", "name", TRACE_STR_COPY(name.c_str())); CHECK_NOT_NULL(platform_); - Debug(this, "Creating isolate for worker with id %llu", thread_id_); + Debug(this, "Creating isolate for worker with id %llu", thread_id_.id); WorkerThreadData data(this); if (isolate_ == nullptr) return; CHECK(data.loop_is_usable()); - Debug(this, "Starting worker with id %llu", thread_id_); + Debug(this, "Starting worker with id %llu", thread_id_.id); { Locker locker(isolate_); Isolate::Scope isolate_scope(isolate_); @@ -317,42 +311,34 @@ void Worker::Run() { CHECK(!context.IsEmpty()); Context::Scope context_scope(context); { - // TODO(addaleax): Use CreateEnvironment(), or generally another - // public API. - env_.reset(new Environment(data.isolate_data_.get(), - context, - std::move(argv_), - std::move(exec_argv_), - Environment::kNoFlags, - thread_id_)); + env_.reset(CreateEnvironment( + data.isolate_data_.get(), + context, + std::move(argv_), + std::move(exec_argv_), + EnvironmentFlags::kNoFlags, + thread_id_)); + if (is_stopped()) return; CHECK_NOT_NULL(env_); env_->set_env_vars(std::move(env_vars_)); - env_->set_abort_on_uncaught_exception(false); - - env_->InitializeLibuv(start_profiler_idle_notifier_); } { Mutex::ScopedLock lock(mutex_); if (stopped_) return; this->env_ = env_.get(); } - Debug(this, "Created Environment for worker with id %llu", thread_id_); + Debug(this, "Created Environment for worker with id %llu", thread_id_.id); if (is_stopped()) return; { - env_->InitializeDiagnostics(); -#if HAVE_INSPECTOR - env_->InitializeInspector(std::move(inspector_parent_handle_)); -#endif - HandleScope handle_scope(isolate_); - - if (!env_->RunBootstrapping().IsEmpty()) { - CreateEnvMessagePort(env_.get()); - if (is_stopped()) return; - Debug(this, "Created message port for worker %llu", thread_id_); - USE(StartExecution(env_.get(), "internal/main/worker_thread")); + CreateEnvMessagePort(env_.get()); + Debug(this, "Created message port for worker %llu", thread_id_.id); + if (LoadEnvironment(env_.get(), + std::move(inspector_parent_handle_)) + .IsEmpty()) { + return; } - Debug(this, "Loaded environment for worker %llu", thread_id_); + Debug(this, "Loaded environment for worker %llu", thread_id_.id); } if (is_stopped()) return; @@ -392,11 +378,11 @@ void Worker::Run() { exit_code_ = exit_code; Debug(this, "Exiting thread for worker %llu with exit code %d", - thread_id_, exit_code_); + thread_id_.id, exit_code_); } } - Debug(this, "Worker %llu thread stops", thread_id_); + Debug(this, "Worker %llu thread stops", thread_id_.id); } void Worker::CreateEnvMessagePort(Environment* env) { @@ -455,7 +441,7 @@ Worker::~Worker() { CHECK_NULL(env_); CHECK(thread_joined_); - Debug(this, "Worker %llu destroyed", thread_id_); + Debug(this, "Worker %llu destroyed", thread_id_.id); } void Worker::New(const FunctionCallbackInfo& args) { @@ -663,7 +649,7 @@ void Worker::StopThread(const FunctionCallbackInfo& args) { Worker* w; ASSIGN_OR_RETURN_UNWRAP(&w, args.This()); - Debug(w, "Worker %llu is getting stopped by parent", w->thread_id_); + Debug(w, "Worker %llu is getting stopped by parent", w->thread_id_.id); w->Exit(1); } @@ -699,7 +685,7 @@ Local Worker::GetResourceLimits(Isolate* isolate) const { void Worker::Exit(int code) { Mutex::ScopedLock lock(mutex_); - Debug(this, "Worker %llu called Exit(%d)", thread_id_, code); + Debug(this, "Worker %llu called Exit(%d)", thread_id_.id, code); if (env_ != nullptr) { exit_code_ = code; Stop(env_); @@ -726,7 +712,7 @@ void Worker::TakeHeapSnapshot(const FunctionCallbackInfo& args) { Worker* w; ASSIGN_OR_RETURN_UNWRAP(&w, args.This()); - Debug(w, "Worker %llu taking heap snapshot", w->thread_id_); + Debug(w, "Worker %llu taking heap snapshot", w->thread_id_.id); Environment* env = w->env(); AsyncHooks::DefaultTriggerAsyncIdScope trigger_id_scope(w); @@ -766,6 +752,7 @@ namespace { void GetEnvMessagePort(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); Local port = env->message_port(); + CHECK_IMPLIES(!env->is_main_thread(), !port.IsEmpty()); if (!port.IsEmpty()) { CHECK_EQ(port->CreationContext()->GetIsolate(), args.GetIsolate()); args.GetReturnValue().Set(port); diff --git a/src/node_worker.h b/src/node_worker.h index 3611a8536fe040..a7d02914826563 100644 --- a/src/node_worker.h +++ b/src/node_worker.h @@ -75,12 +75,9 @@ class Worker : public AsyncWrap { MultiIsolatePlatform* platform_; std::shared_ptr array_buffer_allocator_; v8::Isolate* isolate_ = nullptr; - bool start_profiler_idle_notifier_; uv_thread_t tid_; -#if HAVE_INSPECTOR - std::unique_ptr inspector_parent_handle_; -#endif + std::unique_ptr inspector_parent_handle_; // This mutex protects access to all variables listed below it. mutable Mutex mutex_; @@ -89,7 +86,7 @@ class Worker : public AsyncWrap { const char* custom_error_ = nullptr; std::string custom_error_str_; int exit_code_ = 0; - uint64_t thread_id_ = -1; + ThreadId thread_id_; uintptr_t stack_base_ = 0; // Custom resource constraints: From c5aa3f4adbda0361ca117699f941326c1f9dff02 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 13 Nov 2019 15:54:57 +0000 Subject: [PATCH 404/492] src: provide a variant of LoadEnvironment taking a callback This allows embedders to flexibly control how they start JS code rather than using `third_party_main`. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 7 ++-- src/node.cc | 29 +++++++++++++---- src/node.h | 15 +++++++-- src/node_internals.h | 5 +-- src/node_worker.cc | 1 + test/cctest/test_environment.cc | 58 +++++++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 16 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 2f17e1b2c75100..6154cb395f19b2 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -409,11 +409,12 @@ NODE_EXTERN std::unique_ptr GetInspectorParentHandle( } void LoadEnvironment(Environment* env) { - USE(LoadEnvironment(env, {})); + USE(LoadEnvironment(env, nullptr, {})); } MaybeLocal LoadEnvironment( Environment* env, + StartExecutionCallback cb, std::unique_ptr inspector_parent_handle) { env->InitializeLibuv(per_process::v8_is_profiling); env->InitializeDiagnostics(); @@ -428,9 +429,7 @@ MaybeLocal LoadEnvironment( } #endif - // TODO(joyeecheung): Allow embedders to customize the entry - // point more directly without using _third_party_main.js - return StartExecution(env); + return StartExecution(env, cb); } Environment* GetCurrentEnvironment(Local context) { diff --git a/src/node.cc b/src/node.cc index c34a875768315c..46e8f74cc286f7 100644 --- a/src/node.cc +++ b/src/node.cc @@ -364,6 +364,7 @@ void MarkBootstrapComplete(const FunctionCallbackInfo& args) { performance::NODE_PERFORMANCE_MILESTONE_BOOTSTRAP_COMPLETE); } +static MaybeLocal StartExecution(Environment* env, const char* main_script_id) { EscapableHandleScope scope(env->isolate()); CHECK_NOT_NULL(main_script_id); @@ -384,17 +385,31 @@ MaybeLocal StartExecution(Environment* env, const char* main_script_id) { ->GetFunction(env->context()) .ToLocalChecked()}; - InternalCallbackScope callback_scope( - env, - Object::New(env->isolate()), - { 1, 0 }, - InternalCallbackScope::kSkipAsyncHooks); - return scope.EscapeMaybe( ExecuteBootstrapper(env, main_script_id, ¶meters, &arguments)); } -MaybeLocal StartExecution(Environment* env) { +MaybeLocal StartExecution(Environment* env, StartExecutionCallback cb) { + InternalCallbackScope callback_scope( + env, + Object::New(env->isolate()), + { 1, 0 }, + InternalCallbackScope::kSkipAsyncHooks); + + if (cb != nullptr) { + EscapableHandleScope scope(env->isolate()); + + if (StartExecution(env, "internal/bootstrap/environment").IsEmpty()) + return {}; + + StartExecutionCallbackInfo info = { + env->process_object(), + env->native_module_require(), + }; + + return scope.EscapeMaybe(cb(info)); + } + // To allow people to extend Node in different ways, this hook allows // one to drop a file lib/_third_party_main.js into the build // directory which will be executed instead of Node's normal loading. diff --git a/src/node.h b/src/node.h index 4357291de79cec..e511579b346809 100644 --- a/src/node.h +++ b/src/node.h @@ -73,6 +73,7 @@ #include "node_version.h" // NODE_MODULE_VERSION #include +#include #define NODE_MAKE_VERSION(major, minor, patch) \ ((major) * 0x1000 + (minor) * 0x100 + (patch)) @@ -417,12 +418,20 @@ NODE_EXTERN std::unique_ptr GetInspectorParentHandle( ThreadId child_thread_id, const char* child_url); -// TODO(addaleax): Deprecate this in favour of the MaybeLocal<> overload -// and provide a more flexible approach than third_party_main. +struct StartExecutionCallbackInfo { + v8::Local process_object; + v8::Local native_require; +}; + +using StartExecutionCallback = + std::function(const StartExecutionCallbackInfo&)>; + +// TODO(addaleax): Deprecate this in favour of the MaybeLocal<> overload. NODE_EXTERN void LoadEnvironment(Environment* env); NODE_EXTERN v8::MaybeLocal LoadEnvironment( Environment* env, - std::unique_ptr inspector_parent_handle); + StartExecutionCallback cb, + std::unique_ptr inspector_parent_handle = {}); NODE_EXTERN void FreeEnvironment(Environment* env); // This may return nullptr if context is not associated with a Node instance. diff --git a/src/node_internals.h b/src/node_internals.h index 38de262e05e8d9..25d279e615109b 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -301,9 +301,10 @@ void DefineZlibConstants(v8::Local target); v8::Isolate* NewIsolate(v8::Isolate::CreateParams* params, uv_loop_t* event_loop, MultiIsolatePlatform* platform); -v8::MaybeLocal StartExecution(Environment* env); +// This overload automatically picks the right 'main_script_id' if no callback +// was provided by the embedder. v8::MaybeLocal StartExecution(Environment* env, - const char* main_script_id); + StartExecutionCallback cb = nullptr); v8::MaybeLocal GetPerContextExports(v8::Local context); v8::MaybeLocal ExecuteBootstrapper( Environment* env, diff --git a/src/node_worker.cc b/src/node_worker.cc index dcce464cf80f88..846f4c82c4d26c 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -333,6 +333,7 @@ void Worker::Run() { CreateEnvMessagePort(env_.get()); Debug(this, "Created message port for worker %llu", thread_id_.id); if (LoadEnvironment(env_.get(), + nullptr, std::move(inspector_parent_handle_)) .IsEmpty()) { return; diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 75aa4b7d840e12..ae11f945d777a8 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -51,6 +51,35 @@ class EnvironmentTest : public EnvironmentTestFixture { // CHECK(result->IsString()); // } +TEST_F(EnvironmentTest, LoadEnvironmentWithCallback) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env {handle_scope, argv}; + + v8::Local context = isolate_->GetCurrentContext(); + bool called_cb = false; + node::LoadEnvironment(*env, + [&](const node::StartExecutionCallbackInfo& info) + -> v8::MaybeLocal { + called_cb = true; + + CHECK(info.process_object->IsObject()); + CHECK(info.native_require->IsFunction()); + + v8::Local argv0 = info.process_object->Get( + context, + v8::String::NewFromOneByte( + isolate_, + reinterpret_cast("argv0"), + v8::NewStringType::kNormal).ToLocalChecked()).ToLocalChecked(); + CHECK(argv0->IsString()); + + return info.process_object; + }); + + CHECK(called_cb); +} + TEST_F(EnvironmentTest, AtExitWithEnvironment) { const v8::HandleScope handle_scope(isolate_); const Argv argv; @@ -188,6 +217,35 @@ static void at_exit_js(void* arg) { called_at_exit_js = true; } +TEST_F(EnvironmentTest, SetImmediateCleanup) { + int called = 0; + int called_unref = 0; + + { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env {handle_scope, argv}; + + node::LoadEnvironment(*env, + [&](const node::StartExecutionCallbackInfo& info) + -> v8::MaybeLocal { + return v8::Object::New(isolate_); + }); + + (*env)->SetImmediate([&](node::Environment* env_arg) { + EXPECT_EQ(env_arg, *env); + called++; + }); + (*env)->SetUnrefImmediate([&](node::Environment* env_arg) { + EXPECT_EQ(env_arg, *env); + called_unref++; + }); + } + + EXPECT_EQ(called, 1); + EXPECT_EQ(called_unref, 0); +} + static char hello[] = "hello"; TEST_F(EnvironmentTest, BufferWithFreeCallbackIsDetached) { From 4a6748d2c3470644969f897c31ba6ef11b9fdf08 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 19 Nov 2019 15:42:09 +0100 Subject: [PATCH 405/492] src: add LoadEnvironment() variant taking a string Allow passing a string as the main module rather than using the callback variant. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 36 ++++++++++++++++++++++++++++++++- src/env-inl.h | 5 +++++ src/env.h | 7 +++++++ src/node.h | 4 ++++ src/node_native_module.cc | 8 ++++++++ src/node_native_module.h | 2 ++ src/node_native_module_env.cc | 4 ++++ src/node_native_module_env.h | 1 + src/node_worker.cc | 2 +- test/cctest/test_environment.cc | 27 +++++++++++++++++++++++++ 10 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 6154cb395f19b2..a5d261b7ecd970 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -409,7 +409,9 @@ NODE_EXTERN std::unique_ptr GetInspectorParentHandle( } void LoadEnvironment(Environment* env) { - USE(LoadEnvironment(env, nullptr, {})); + USE(LoadEnvironment(env, + StartExecutionCallback{}, + {})); } MaybeLocal LoadEnvironment( @@ -432,6 +434,38 @@ MaybeLocal LoadEnvironment( return StartExecution(env, cb); } +MaybeLocal LoadEnvironment( + Environment* env, + const char* main_script_source_utf8, + std::unique_ptr inspector_parent_handle) { + CHECK_NOT_NULL(main_script_source_utf8); + return LoadEnvironment( + env, + [&](const StartExecutionCallbackInfo& info) -> MaybeLocal { + // This is a slightly hacky way to convert UTF-8 to UTF-16. + Local str = + String::NewFromUtf8(env->isolate(), + main_script_source_utf8, + v8::NewStringType::kNormal).ToLocalChecked(); + auto main_utf16 = std::make_unique(env->isolate(), str); + + // TODO(addaleax): Avoid having a global table for all scripts. + std::string name = "embedder_main_" + std::to_string(env->thread_id()); + native_module::NativeModuleEnv::Add( + name.c_str(), + UnionBytes(**main_utf16, main_utf16->length())); + env->set_main_utf16(std::move(main_utf16)); + std::vector> params = { + env->process_string(), + env->require_string()}; + std::vector> args = { + env->process_object(), + env->native_module_require()}; + return ExecuteBootstrapper(env, name.c_str(), ¶ms, &args); + }, + std::move(inspector_parent_handle)); +} + Environment* GetCurrentEnvironment(Local context) { return Environment::GetCurrent(context); } diff --git a/src/env-inl.h b/src/env-inl.h index 598b84c07ca8d2..cb10b4d9554e23 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -1216,6 +1216,11 @@ int64_t Environment::base_object_count() const { return base_object_count_; } +void Environment::set_main_utf16(std::unique_ptr str) { + CHECK(!main_utf16_); + main_utf16_ = std::move(str); +} + #define VP(PropertyName, StringValue) V(v8::Private, PropertyName) #define VY(PropertyName, StringValue) V(v8::Symbol, PropertyName) #define VS(PropertyName, StringValue) V(v8::String, PropertyName) diff --git a/src/env.h b/src/env.h index 8331a79dbc9918..0566c2d26dc4f2 100644 --- a/src/env.h +++ b/src/env.h @@ -1269,6 +1269,8 @@ class Environment : public MemoryRetainer { void AddArrayBufferAllocatorToKeepAliveUntilIsolateDispose( std::shared_ptr); + inline void set_main_utf16(std::unique_ptr); + private: inline void ThrowError(v8::Local (*fun)(v8::Local), const char* errmsg); @@ -1435,6 +1437,11 @@ class Environment : public MemoryRetainer { #undef V v8::Global context_; + + // Keeps the main script source alive is one was passed to LoadEnvironment(). + // We should probably find a way to just use plain `v8::String`s created from + // the source passed to LoadEnvironment() directly instead. + std::unique_ptr main_utf16_; }; } // namespace node diff --git a/src/node.h b/src/node.h index e511579b346809..2e00d043c13ff6 100644 --- a/src/node.h +++ b/src/node.h @@ -432,6 +432,10 @@ NODE_EXTERN v8::MaybeLocal LoadEnvironment( Environment* env, StartExecutionCallback cb, std::unique_ptr inspector_parent_handle = {}); +NODE_EXTERN v8::MaybeLocal LoadEnvironment( + Environment* env, + const char* main_script_source_utf8, + std::unique_ptr inspector_parent_handle = {}); NODE_EXTERN void FreeEnvironment(Environment* env); // This may return nullptr if context is not associated with a Node instance. diff --git a/src/node_native_module.cc b/src/node_native_module.cc index 7362207412efa4..74729c412674be 100644 --- a/src/node_native_module.cc +++ b/src/node_native_module.cc @@ -30,6 +30,14 @@ bool NativeModuleLoader::Exists(const char* id) { return source_.find(id) != source_.end(); } +bool NativeModuleLoader::Add(const char* id, const UnionBytes& source) { + if (Exists(id)) { + return false; + } + source_.emplace(id, source); + return true; +} + Local NativeModuleLoader::GetSourceObject(Local context) { Isolate* isolate = context->GetIsolate(); Local out = Object::New(isolate); diff --git a/src/node_native_module.h b/src/node_native_module.h index c0bce3bce42c84..3be3f2364dd252 100644 --- a/src/node_native_module.h +++ b/src/node_native_module.h @@ -47,6 +47,8 @@ class NativeModuleLoader { UnionBytes GetConfig(); // Return data for config.gypi bool Exists(const char* id); + bool Add(const char* id, const UnionBytes& source); + v8::Local GetSourceObject(v8::Local context); v8::Local GetConfigString(v8::Isolate* isolate); std::vector GetModuleIds(); diff --git a/src/node_native_module_env.cc b/src/node_native_module_env.cc index 9a6ccf99313cf8..cdd98e8220eb46 100644 --- a/src/node_native_module_env.cc +++ b/src/node_native_module_env.cc @@ -32,6 +32,10 @@ Local ToJsSet(Local context, const std::set& in) { return out; } +bool NativeModuleEnv::Add(const char* id, const UnionBytes& source) { + return NativeModuleLoader::GetInstance()->Add(id, source); +} + bool NativeModuleEnv::Exists(const char* id) { return NativeModuleLoader::GetInstance()->Exists(id); } diff --git a/src/node_native_module_env.h b/src/node_native_module_env.h index f662c67be50d40..bc36be75109639 100644 --- a/src/node_native_module_env.h +++ b/src/node_native_module_env.h @@ -29,6 +29,7 @@ class NativeModuleEnv { // Returns config.gypi as a JSON string static v8::Local GetConfigString(v8::Isolate* isolate); static bool Exists(const char* id); + static bool Add(const char* id, const UnionBytes& source); // Loads data into NativeModuleLoader::.instance.code_cache_ // Generated by mkcodecache as node_code_cache.cc when diff --git a/src/node_worker.cc b/src/node_worker.cc index 846f4c82c4d26c..a61bf51f1c9a02 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -333,7 +333,7 @@ void Worker::Run() { CreateEnvMessagePort(env_.get()); Debug(this, "Created message port for worker %llu", thread_id_.id); if (LoadEnvironment(env_.get(), - nullptr, + StartExecutionCallback{}, std::move(inspector_parent_handle_)) .IsEmpty()) { return; diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index ae11f945d777a8..57308096f4dc22 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -80,6 +80,33 @@ TEST_F(EnvironmentTest, LoadEnvironmentWithCallback) { CHECK(called_cb); } +TEST_F(EnvironmentTest, LoadEnvironmentWithSource) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env {handle_scope, argv}; + + v8::Local context = isolate_->GetCurrentContext(); + v8::Local main_ret = + node::LoadEnvironment(*env, + "return { process, require };").ToLocalChecked(); + + CHECK(main_ret->IsObject()); + CHECK(main_ret.As()->Get( + context, + v8::String::NewFromOneByte( + isolate_, + reinterpret_cast("process"), + v8::NewStringType::kNormal).ToLocalChecked()) + .ToLocalChecked()->IsObject()); + CHECK(main_ret.As()->Get( + context, + v8::String::NewFromOneByte( + isolate_, + reinterpret_cast("require"), + v8::NewStringType::kNormal).ToLocalChecked()) + .ToLocalChecked()->IsFunction()); +} + TEST_F(EnvironmentTest, AtExitWithEnvironment) { const v8::HandleScope handle_scope(isolate_); const Argv argv; From f325c9544f54c5c4fae624ec5fa1d570d3dc0b80 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 14:20:00 -0800 Subject: [PATCH 406/492] test: re-enable cctest that was commented out Refs: https://github.com/nodejs/node/pull/31910 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- test/cctest/test_environment.cc | 39 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 57308096f4dc22..7f6dbb8d9d9cfd 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -32,24 +32,27 @@ class EnvironmentTest : public EnvironmentTestFixture { } }; -// TODO(codebytere): re-enable this test. -// TEST_F(EnvironmentTest, PreExeuctionPreparation) { -// const v8::HandleScope handle_scope(isolate_); -// const Argv argv; -// Env env {handle_scope, argv}; - -// v8::Local context = isolate_->GetCurrentContext(); - -// const char* run_script = "process.argv0"; -// v8::Local script = v8::Script::Compile( -// context, -// v8::String::NewFromOneByte(isolate_, -// reinterpret_cast(run_script), -// v8::NewStringType::kNormal).ToLocalChecked()) -// .ToLocalChecked(); -// v8::Local result = script->Run(context).ToLocalChecked(); -// CHECK(result->IsString()); -// } +TEST_F(EnvironmentTest, PreExecutionPreparation) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env {handle_scope, argv}; + + node::LoadEnvironment(*env, [&](const node::StartExecutionCallbackInfo& info) + -> v8::MaybeLocal { + return v8::Null(isolate_); + }); + + v8::Local context = isolate_->GetCurrentContext(); + const char* run_script = "process.argv0"; + v8::Local script = v8::Script::Compile( + context, + v8::String::NewFromOneByte(isolate_, + reinterpret_cast(run_script), + v8::NewStringType::kNormal).ToLocalChecked()) + .ToLocalChecked(); + v8::Local result = script->Run(context).ToLocalChecked(); + CHECK(result->IsString()); +} TEST_F(EnvironmentTest, LoadEnvironmentWithCallback) { const v8::HandleScope handle_scope(isolate_); From 8005e637b166d0649616b4f9bdd7a709f9260a83 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 17:11:35 -0800 Subject: [PATCH 407/492] src: add unique_ptr equivalent of CreatePlatform This makes this bit of the embedder situation a bit easier to use. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 9 ++++++++- src/node.h | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index a5d261b7ecd970..81be9f7dfeaa28 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -477,13 +477,20 @@ MultiIsolatePlatform* GetMainThreadMultiIsolatePlatform() { MultiIsolatePlatform* CreatePlatform( int thread_pool_size, node::tracing::TracingController* tracing_controller) { - return new NodePlatform(thread_pool_size, tracing_controller); + return MultiIsolatePlatform::Create(thread_pool_size, tracing_controller) + .release(); } void FreePlatform(MultiIsolatePlatform* platform) { delete platform; } +std::unique_ptr MultiIsolatePlatform::Create( + int thread_pool_size, + node::tracing::TracingController* tracing_controller) { + return std::make_unique(thread_pool_size, tracing_controller); +} + MaybeLocal GetPerContextExports(Local context) { Isolate* isolate = context->GetIsolate(); EscapableHandleScope handle_scope(isolate); diff --git a/src/node.h b/src/node.h index 2e00d043c13ff6..10a64744450bc8 100644 --- a/src/node.h +++ b/src/node.h @@ -298,6 +298,10 @@ class NODE_EXTERN MultiIsolatePlatform : public v8::Platform { virtual void AddIsolateFinishedCallback(v8::Isolate* isolate, void (*callback)(void*), void* data) = 0; + + static std::unique_ptr Create( + int thread_pool_size, + node::tracing::TracingController* tracing_controller = nullptr); }; enum IsolateSettingsFlags { @@ -446,6 +450,7 @@ NODE_EXTERN Environment* GetCurrentEnvironment(v8::Local context); // it returns nullptr. NODE_EXTERN MultiIsolatePlatform* GetMainThreadMultiIsolatePlatform(); +// Legacy variants of MultiIsolatePlatform::Create(). NODE_EXTERN MultiIsolatePlatform* CreatePlatform( int thread_pool_size, node::tracing::TracingController* tracing_controller); From a770a35f612672394bc456c2fec336e2c6e6fbed Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 17:40:54 -0800 Subject: [PATCH 408/492] src: make InitializeNodeWithArgs() official public API This is a decent replacement for the to-be-deprecated Init() API. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/node.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/node.h b/src/node.h index 10a64744450bc8..7a8709f7d624ea 100644 --- a/src/node.h +++ b/src/node.h @@ -223,10 +223,20 @@ NODE_EXTERN int Stop(Environment* env); // TODO(addaleax): Officially deprecate this and replace it with something // better suited for a public embedder API. +// It is recommended to use InitializeNodeWithArgs() instead as an embedder. +// Init() calls InitializeNodeWithArgs() and exits the process with the exit +// code returned from it. NODE_EXTERN void Init(int* argc, const char** argv, int* exec_argc, const char*** exec_argv); +// Set up per-process state needed to run Node.js. This will consume arguments +// from argv, fill exec_argv, and possibly add errors resulting from parsing +// the arguments to `errors`. The return value is a suggested exit code for the +// program; If it is 0, then initializing Node.js succeeded. +NODE_EXTERN int InitializeNodeWithArgs(std::vector* argv, + std::vector* exec_argv, + std::vector* errors); enum OptionEnvvarSettings { kAllowedInEnvironment, From 9b84ee64801dadc826c3d1f2e5063aeeeafd561e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 21:13:59 -0800 Subject: [PATCH 409/492] src: add ability to look up platform based on `Environment*` This should eventually remove any necessity to use the global-state `GetMainThreadMultiIsolatePlatform()`. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 8 ++++++++ src/node.h | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 81be9f7dfeaa28..a4b688dde7e17a 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -474,6 +474,14 @@ MultiIsolatePlatform* GetMainThreadMultiIsolatePlatform() { return per_process::v8_platform.Platform(); } +MultiIsolatePlatform* GetMultiIsolatePlatform(Environment* env) { + return GetMultiIsolatePlatform(env->isolate_data()); +} + +MultiIsolatePlatform* GetMultiIsolatePlatform(IsolateData* env) { + return env->platform(); +} + MultiIsolatePlatform* CreatePlatform( int thread_pool_size, node::tracing::TracingController* tracing_controller) { diff --git a/src/node.h b/src/node.h index 7a8709f7d624ea..023e60dafd615e 100644 --- a/src/node.h +++ b/src/node.h @@ -456,9 +456,14 @@ NODE_EXTERN void FreeEnvironment(Environment* env); NODE_EXTERN Environment* GetCurrentEnvironment(v8::Local context); // This returns the MultiIsolatePlatform used in the main thread of Node.js. -// If NODE_USE_V8_PLATFORM haven't been defined when Node.js was built, +// If NODE_USE_V8_PLATFORM has not been defined when Node.js was built, // it returns nullptr. +// TODO(addaleax): Deprecate in favour of GetMultiIsolatePlatform(). NODE_EXTERN MultiIsolatePlatform* GetMainThreadMultiIsolatePlatform(); +// This returns the MultiIsolatePlatform used for an Environment or IsolateData +// instance, if one exists. +NODE_EXTERN MultiIsolatePlatform* GetMultiIsolatePlatform(Environment* env); +NODE_EXTERN MultiIsolatePlatform* GetMultiIsolatePlatform(IsolateData* env); // Legacy variants of MultiIsolatePlatform::Create(). NODE_EXTERN MultiIsolatePlatform* CreatePlatform( From 2f8f76736bf14e8ac88ae8bac2f22aa312bf1b6b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 21:43:02 -0800 Subject: [PATCH 410/492] src: allow non-Node.js TracingControllers We do not need a Node.js-provided `v8::TracingController`, generally. Loosen that restriction in order to make it easier for embedders to provide their own subclass of `v8::TracingController`, or none at all. Refs: https://github.com/electron/electron/commit/9c36576dddfaecde1298ff3e089d21a6e54fe67f Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/api/environment.cc | 10 +++++++++- src/node.h | 6 +++++- src/node_platform.cc | 13 ++++++++----- src/node_platform.h | 6 +++--- src/tracing/agent.cc | 5 +++-- src/tracing/trace_event.cc | 10 ++++++++-- src/tracing/trace_event.h | 11 +++++++---- 7 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index a4b688dde7e17a..27eb47328c510c 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -485,6 +485,14 @@ MultiIsolatePlatform* GetMultiIsolatePlatform(IsolateData* env) { MultiIsolatePlatform* CreatePlatform( int thread_pool_size, node::tracing::TracingController* tracing_controller) { + return CreatePlatform( + thread_pool_size, + static_cast(tracing_controller)); +} + +MultiIsolatePlatform* CreatePlatform( + int thread_pool_size, + v8::TracingController* tracing_controller) { return MultiIsolatePlatform::Create(thread_pool_size, tracing_controller) .release(); } @@ -495,7 +503,7 @@ void FreePlatform(MultiIsolatePlatform* platform) { std::unique_ptr MultiIsolatePlatform::Create( int thread_pool_size, - node::tracing::TracingController* tracing_controller) { + v8::TracingController* tracing_controller) { return std::make_unique(thread_pool_size, tracing_controller); } diff --git a/src/node.h b/src/node.h index 023e60dafd615e..cb3ff139278ecd 100644 --- a/src/node.h +++ b/src/node.h @@ -311,7 +311,7 @@ class NODE_EXTERN MultiIsolatePlatform : public v8::Platform { static std::unique_ptr Create( int thread_pool_size, - node::tracing::TracingController* tracing_controller = nullptr); + v8::TracingController* tracing_controller = nullptr); }; enum IsolateSettingsFlags { @@ -466,9 +466,13 @@ NODE_EXTERN MultiIsolatePlatform* GetMultiIsolatePlatform(Environment* env); NODE_EXTERN MultiIsolatePlatform* GetMultiIsolatePlatform(IsolateData* env); // Legacy variants of MultiIsolatePlatform::Create(). +// TODO(addaleax): Deprecate in favour of the v8::TracingController variant. NODE_EXTERN MultiIsolatePlatform* CreatePlatform( int thread_pool_size, node::tracing::TracingController* tracing_controller); +NODE_EXTERN MultiIsolatePlatform* CreatePlatform( + int thread_pool_size, + v8::TracingController* tracing_controller); NODE_EXTERN void FreePlatform(MultiIsolatePlatform* platform); NODE_EXTERN void EmitBeforeExit(Environment* env); diff --git a/src/node_platform.cc b/src/node_platform.cc index e6e880b87b5380..35444d45506823 100644 --- a/src/node_platform.cc +++ b/src/node_platform.cc @@ -13,7 +13,6 @@ using v8::Isolate; using v8::Object; using v8::Platform; using v8::Task; -using node::tracing::TracingController; namespace { @@ -320,12 +319,16 @@ void PerIsolatePlatformData::DecreaseHandleCount() { } NodePlatform::NodePlatform(int thread_pool_size, - TracingController* tracing_controller) { - if (tracing_controller) { + v8::TracingController* tracing_controller) { + if (tracing_controller != nullptr) { tracing_controller_ = tracing_controller; } else { - tracing_controller_ = new TracingController(); + tracing_controller_ = new v8::TracingController(); } + // TODO(addaleax): It's a bit icky that we use global state here, but we can't + // really do anything about it unless V8 starts exposing a way to access the + // current v8::Platform instance. + tracing::TraceEventHelper::SetTracingController(tracing_controller_); worker_thread_task_runner_ = std::make_shared(thread_pool_size); } @@ -490,7 +493,7 @@ double NodePlatform::CurrentClockTimeMillis() { return SystemClockTimeMillis(); } -TracingController* NodePlatform::GetTracingController() { +v8::TracingController* NodePlatform::GetTracingController() { CHECK_NOT_NULL(tracing_controller_); return tracing_controller_; } diff --git a/src/node_platform.h b/src/node_platform.h index 533ae1bcca8248..3cf526ae3da124 100644 --- a/src/node_platform.h +++ b/src/node_platform.h @@ -137,7 +137,7 @@ class WorkerThreadsTaskRunner { class NodePlatform : public MultiIsolatePlatform { public: NodePlatform(int thread_pool_size, - node::tracing::TracingController* tracing_controller); + v8::TracingController* tracing_controller); ~NodePlatform() override = default; void DrainTasks(v8::Isolate* isolate) override; @@ -159,7 +159,7 @@ class NodePlatform : public MultiIsolatePlatform { bool IdleTasksEnabled(v8::Isolate* isolate) override; double MonotonicallyIncreasingTime() override; double CurrentClockTimeMillis() override; - node::tracing::TracingController* GetTracingController() override; + v8::TracingController* GetTracingController() override; bool FlushForegroundTasks(v8::Isolate* isolate) override; void RegisterIsolate(v8::Isolate* isolate, uv_loop_t* loop) override; @@ -179,7 +179,7 @@ class NodePlatform : public MultiIsolatePlatform { std::unordered_map> per_isolate_; - node::tracing::TracingController* tracing_controller_; + v8::TracingController* tracing_controller_; std::shared_ptr worker_thread_task_runner_; }; diff --git a/src/tracing/agent.cc b/src/tracing/agent.cc index 7d265dcb0c4c3b..7ce59674356f97 100644 --- a/src/tracing/agent.cc +++ b/src/tracing/agent.cc @@ -242,8 +242,9 @@ void TracingController::AddMetadataEvent( TRACE_EVENT_FLAG_NONE, CurrentTimestampMicroseconds(), CurrentCpuTimestampMicroseconds()); - node::tracing::TraceEventHelper::GetAgent()->AddMetadataEvent( - std::move(trace_event)); + Agent* node_agent = node::tracing::TraceEventHelper::GetAgent(); + if (node_agent != nullptr) + node_agent->AddMetadataEvent(std::move(trace_event)); } } // namespace tracing diff --git a/src/tracing/trace_event.cc b/src/tracing/trace_event.cc index 9232c34c4ca322..da562c1bf7f8ff 100644 --- a/src/tracing/trace_event.cc +++ b/src/tracing/trace_event.cc @@ -4,17 +4,23 @@ namespace node { namespace tracing { Agent* g_agent = nullptr; +v8::TracingController* g_controller = nullptr; void TraceEventHelper::SetAgent(Agent* agent) { g_agent = agent; + g_controller = agent->GetTracingController(); } Agent* TraceEventHelper::GetAgent() { return g_agent; } -TracingController* TraceEventHelper::GetTracingController() { - return g_agent->GetTracingController(); +v8::TracingController* TraceEventHelper::GetTracingController() { + return g_controller; +} + +void TraceEventHelper::SetTracingController(v8::TracingController* controller) { + g_controller = controller; } } // namespace tracing diff --git a/src/tracing/trace_event.h b/src/tracing/trace_event.h index c11544dd42cdd0..2a79c5bc05b501 100644 --- a/src/tracing/trace_event.h +++ b/src/tracing/trace_event.h @@ -314,7 +314,9 @@ const uint64_t kNoId = 0; // Refs: https://github.com/nodejs/node/pull/28724 class NODE_EXTERN TraceEventHelper { public: - static TracingController* GetTracingController(); + static v8::TracingController* GetTracingController(); + static void SetTracingController(v8::TracingController* controller); + static Agent* GetAgent(); static void SetAgent(Agent* agent); @@ -514,9 +516,10 @@ static V8_INLINE void AddMetadataEventImpl( arg_convertibles[1].reset(reinterpret_cast( static_cast(arg_values[1]))); } - node::tracing::TracingController* controller = - node::tracing::TraceEventHelper::GetTracingController(); - return controller->AddMetadataEvent( + node::tracing::Agent* agent = + node::tracing::TraceEventHelper::GetAgent(); + if (agent == nullptr) return; + return agent->GetTracingController()->AddMetadataEvent( category_group_enabled, name, num_args, arg_names, arg_types, arg_values, arg_convertibles, flags); } From a28c9900613f68f92f59ea3881b68c71d8e0010f Mon Sep 17 00:00:00 2001 From: Jichan Date: Wed, 8 Jan 2020 21:36:05 +0900 Subject: [PATCH 411/492] src: fix what a dispose without checking If created platform with CreatePlatform, the crash occurs because it does not check if it was initialized to v8_platform when DisposePlatform was called. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 Refs: https://github.com/nodejs/node/pull/31260 Co-authored-by: Anna Henningsen PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/node_v8_platform-inl.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/node_v8_platform-inl.h b/src/node_v8_platform-inl.h index 0f4a98a98551ba..ce6d5dd223d2cb 100644 --- a/src/node_v8_platform-inl.h +++ b/src/node_v8_platform-inl.h @@ -12,6 +12,7 @@ #include "tracing/node_trace_writer.h" #include "tracing/trace_event.h" #include "tracing/traced_value.h" +#include "util.h" namespace node { @@ -79,8 +80,12 @@ class NodeTraceStateObserver }; struct V8Platform { + bool initialized_ = false; + #if NODE_USE_V8_PLATFORM inline void Initialize(int thread_pool_size) { + CHECK(!initialized_); + initialized_ = true; tracing_agent_ = std::make_unique(); node::tracing::TraceEventHelper::SetAgent(tracing_agent_.get()); node::tracing::TracingController* controller = @@ -99,6 +104,10 @@ struct V8Platform { } inline void Dispose() { + if (!initialized_) + return; + initialized_ = false; + StopTracingAgent(); platform_->Shutdown(); delete platform_; From b8c9048a87c30f62b333e57697fa45a4b6c01403 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 22:02:12 -0800 Subject: [PATCH 412/492] src: shutdown platform from FreePlatform() There is currently no way to properly do this. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- src/node_platform.cc | 6 ++++++ src/node_platform.h | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/node_platform.cc b/src/node_platform.cc index 35444d45506823..f7eefc93f8fece 100644 --- a/src/node_platform.cc +++ b/src/node_platform.cc @@ -333,6 +333,10 @@ NodePlatform::NodePlatform(int thread_pool_size, std::make_shared(thread_pool_size); } +NodePlatform::~NodePlatform() { + Shutdown(); +} + void NodePlatform::RegisterIsolate(Isolate* isolate, uv_loop_t* loop) { Mutex::ScopedLock lock(per_isolate_mutex_); std::shared_ptr existing = per_isolate_[isolate]; @@ -362,6 +366,8 @@ void NodePlatform::AddIsolateFinishedCallback(Isolate* isolate, } void NodePlatform::Shutdown() { + if (has_shut_down_) return; + has_shut_down_ = true; worker_thread_task_runner_->Shutdown(); { diff --git a/src/node_platform.h b/src/node_platform.h index 3cf526ae3da124..48ced3eac5b5de 100644 --- a/src/node_platform.h +++ b/src/node_platform.h @@ -138,7 +138,7 @@ class NodePlatform : public MultiIsolatePlatform { public: NodePlatform(int thread_pool_size, v8::TracingController* tracing_controller); - ~NodePlatform() override = default; + ~NodePlatform() override; void DrainTasks(v8::Isolate* isolate) override; void Shutdown(); @@ -181,6 +181,7 @@ class NodePlatform : public MultiIsolatePlatform { v8::TracingController* tracing_controller_; std::shared_ptr worker_thread_task_runner_; + bool has_shut_down_ = false; }; } // namespace node From 4af336d7414d2933b48626f040a7c1153fa45adf Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 18:54:13 -0800 Subject: [PATCH 413/492] src,test: add full-featured embedder API test Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- Makefile | 10 ++- node.gyp | 56 +++++++++++++++ src/node_code_cache_stub.cc | 4 ++ src/node_snapshot_stub.cc | 4 ++ test/embedding/embedtest.cc | 135 ++++++++++++++++++++++++++++++++++++ test/embedding/test.js | 19 +++++ 6 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 test/embedding/embedtest.cc create mode 100644 test/embedding/test.js diff --git a/Makefile b/Makefile index 68e222656eb980..35eeeb618b7097 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,8 @@ coverage-clean: $(RM) out/$(BUILDTYPE)/obj.target/node/src/tracing/*.gcno $(RM) out/$(BUILDTYPE)/obj.target/cctest/src/*.gcno $(RM) out/$(BUILDTYPE)/obj.target/cctest/test/cctest/*.gcno + $(RM) out/$(BUILDTYPE)/obj.target/embedtest/src/*.gcno + $(RM) out/$(BUILDTYPE)/obj.target/embedtest/test/embedding/*.gcno .PHONY: coverage # Build and test with code coverage reporting. Leave the lib directory @@ -250,8 +252,8 @@ coverage-test: coverage-build TEST_CI_ARGS="$(TEST_CI_ARGS) --type=coverage" $(MAKE) $(COVTESTS) $(MAKE) coverage-report-js -(cd out && "../gcovr/scripts/gcovr" \ - --gcov-exclude='.*\b(deps|usr|out|cctest)\b' -v -r Release/obj.target \ - --html --html-detail -o ../coverage/cxxcoverage.html \ + --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v \ + -r Release/obj.target --html --html-detail -o ../coverage/cxxcoverage.html \ --gcov-executable="$(GCOV)") @echo -n "Javascript coverage %: " @grep -B1 Lines coverage/index.html | head -n1 \ @@ -276,6 +278,7 @@ coverage-report-js: # Runs the C++ tests using the built `cctest` executable. cctest: all @out/$(BUILDTYPE)/$@ --gtest_filter=$(GTEST_FILTER) + @out/$(BUILDTYPE)/embedtest "require('./test/embedding/test.js')" .PHONY: list-gtests list-gtests: @@ -531,6 +534,7 @@ test-ci: | clear-stalled bench-addons-build build-addons build-js-native-api-tes $(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \ --mode=$(BUILDTYPE_LOWER) --flaky-tests=$(FLAKY_TESTS) \ $(TEST_CI_ARGS) $(CI_JS_SUITES) $(CI_NATIVE_SUITES) $(CI_DOC) + out/Release/embedtest 'require("./test/embedding/test.js")' @echo "Clean up any leftover processes, error if found." ps awwx | grep Release/node | grep -v grep | cat @PS_OUT=`ps awwx | grep Release/node | grep -v grep | awk '{print $$1}'`; \ @@ -1274,6 +1278,8 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \ test/addons/*/*.h \ test/cctest/*.cc \ test/cctest/*.h \ + test/embedding/*.cc \ + test/embedding/*.h \ test/js-native-api/*/*.cc \ test/js-native-api/*/*.h \ test/node-api/*/*.cc \ diff --git a/node.gyp b/node.gyp index 8173a1afdda8ee..c03623846a2b6e 100644 --- a/node.gyp +++ b/node.gyp @@ -1236,6 +1236,62 @@ ], }, # cctest + { + 'target_name': 'embedtest', + 'type': 'executable', + + 'dependencies': [ + '<(node_lib_target_name)', + 'deps/histogram/histogram.gyp:histogram', + 'deps/uvwasi/uvwasi.gyp:uvwasi', + 'node_dtrace_header', + 'node_dtrace_ustack', + 'node_dtrace_provider', + ], + + 'includes': [ + 'node.gypi' + ], + + 'include_dirs': [ + 'src', + 'tools/msvs/genfiles', + 'deps/v8/include', + 'deps/cares/include', + 'deps/uv/include', + 'deps/uvwasi/include', + 'test/embedding', + ], + + 'sources': [ + 'src/node_snapshot_stub.cc', + 'src/node_code_cache_stub.cc', + 'test/embedding/embedtest.cc', + ], + + 'conditions': [ + ['OS=="solaris"', { + 'ldflags': [ '-I<(SHARED_INTERMEDIATE_DIR)' ] + }], + # Skip cctest while building shared lib node for Windows + [ 'OS=="win" and node_shared=="true"', { + 'type': 'none', + }], + [ 'node_shared=="true"', { + 'xcode_settings': { + 'OTHER_LDFLAGS': [ '-Wl,-rpath,@loader_path', ], + }, + }], + ['OS=="win"', { + 'libraries': [ + 'Dbghelp.lib', + 'winmm.lib', + 'Ws2_32.lib', + ], + }], + ], + }, # embedtest + # TODO(joyeecheung): do not depend on node_lib, # instead create a smaller static library node_lib_base that does # just enough for node_native_module.cc and the cache builder to diff --git a/src/node_code_cache_stub.cc b/src/node_code_cache_stub.cc index 3851508170f812..9d9901738f63b0 100644 --- a/src/node_code_cache_stub.cc +++ b/src/node_code_cache_stub.cc @@ -1,3 +1,7 @@ +// This file is part of the embedder test, which is intentionally built without +// NODE_WANT_INTERNALS, so we define it here manually. +#define NODE_WANT_INTERNALS 1 + #include "node_native_module_env.h" // The stub here is used when configure is run without `--code-cache-path`. diff --git a/src/node_snapshot_stub.cc b/src/node_snapshot_stub.cc index 91bc37121d61fe..fac03b0c87af5d 100644 --- a/src/node_snapshot_stub.cc +++ b/src/node_snapshot_stub.cc @@ -1,3 +1,7 @@ +// This file is part of the embedder test, which is intentionally built without +// NODE_WANT_INTERNALS, so we define it here manually. +#define NODE_WANT_INTERNALS 1 + #include "node_main_instance.h" namespace node { diff --git a/test/embedding/embedtest.cc b/test/embedding/embedtest.cc new file mode 100644 index 00000000000000..366c8f12e2c11f --- /dev/null +++ b/test/embedding/embedtest.cc @@ -0,0 +1,135 @@ +#include "node.h" +#include "uv.h" +#include + +// Note: This file is being referred to from doc/api/embedding.md, and excerpts +// from it are included in the documentation. Try to keep these in sync. + +using node::ArrayBufferAllocator; +using node::Environment; +using node::IsolateData; +using node::MultiIsolatePlatform; +using v8::Context; +using v8::HandleScope; +using v8::Isolate; +using v8::Local; +using v8::Locker; +using v8::MaybeLocal; +using v8::SealHandleScope; +using v8::Value; +using v8::V8; + +static int RunNodeInstance(MultiIsolatePlatform* platform, + const std::vector& args, + const std::vector& exec_args); + +int main(int argc, char** argv) { + std::vector args(argv, argv + argc); + std::vector exec_args; + std::vector errors; + int exit_code = node::InitializeNodeWithArgs(&args, &exec_args, &errors); + for (const std::string& error : errors) + fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str()); + if (exit_code != 0) { + return exit_code; + } + + std::unique_ptr platform = + MultiIsolatePlatform::Create(4); + V8::InitializePlatform(platform.get()); + V8::Initialize(); + + int ret = RunNodeInstance(platform.get(), args, exec_args); + + V8::Dispose(); + V8::ShutdownPlatform(); + return ret; +} + +int RunNodeInstance(MultiIsolatePlatform* platform, + const std::vector& args, + const std::vector& exec_args) { + int exit_code = 0; + uv_loop_t loop; + int ret = uv_loop_init(&loop); + if (ret != 0) { + fprintf(stderr, "%s: Failed to initialize loop: %s\n", + args[0].c_str(), + uv_err_name(ret)); + return 1; + } + + std::shared_ptr allocator = + ArrayBufferAllocator::Create(); + + Isolate* isolate = NewIsolate(allocator.get(), &loop, platform); + if (isolate == nullptr) { + fprintf(stderr, "%s: Failed to initialize V8 Isolate\n", args[0].c_str()); + return 1; + } + + { + Locker locker(isolate); + Isolate::Scope isolate_scope(isolate); + + std::unique_ptr isolate_data( + node::CreateIsolateData(isolate, &loop, platform, allocator.get()), + node::FreeIsolateData); + + HandleScope handle_scope(isolate); + Local context = node::NewContext(isolate); + if (context.IsEmpty()) { + fprintf(stderr, "%s: Failed to initialize V8 Context\n", args[0].c_str()); + return 1; + } + + Context::Scope context_scope(context); + std::unique_ptr env( + node::CreateEnvironment(isolate_data.get(), context, args, exec_args), + node::FreeEnvironment); + + MaybeLocal loadenv_ret = node::LoadEnvironment( + env.get(), + "const publicRequire =" + " require('module').createRequire(process.cwd() + '/');" + "globalThis.require = publicRequire;" + "require('vm').runInThisContext(process.argv[1]);"); + + if (loadenv_ret.IsEmpty()) // There has been a JS exception. + return 1; + + { + SealHandleScope seal(isolate); + bool more; + do { + uv_run(&loop, UV_RUN_DEFAULT); + + platform->DrainTasks(isolate); + more = uv_loop_alive(&loop); + if (more) continue; + + node::EmitBeforeExit(env.get()); + more = uv_loop_alive(&loop); + } while (more == true); + } + + exit_code = node::EmitExit(env.get()); + + node::Stop(env.get()); + } + + bool platform_finished = false; + platform->AddIsolateFinishedCallback(isolate, [](void* data) { + *static_cast(data) = true; + }, &platform_finished); + platform->UnregisterIsolate(isolate); + isolate->Dispose(); + + // Wait until the platform has cleaned up all relevant resources. + while (!platform_finished) + uv_run(&loop, UV_RUN_ONCE); + int err = uv_loop_close(&loop); + assert(err == 0); + + return exit_code; +} diff --git a/test/embedding/test.js b/test/embedding/test.js new file mode 100644 index 00000000000000..a802de1849021f --- /dev/null +++ b/test/embedding/test.js @@ -0,0 +1,19 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const child_process = require('child_process'); + +common.allowGlobals(global.require); + +assert.strictEqual( + child_process.spawnSync(process.execPath, ['console.log(42)']) + .stdout.toString().trim(), + '42'); + +assert.strictEqual( + child_process.spawnSync(process.execPath, ['throw new Error()']).status, + 1); + +assert.strictEqual( + child_process.spawnSync(process.execPath, ['process.exitCode = 8']).status, + 8); From 1a7082052fb35ec9a895892cdd60c213b14a82f6 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 27 Feb 2020 23:14:33 -0800 Subject: [PATCH 414/492] doc: add basic embedding example documentation Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- doc/api/embedding.md | 226 +++++++++++++++++++++++++++++++++++++++++++ doc/api/index.md | 7 +- 2 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 doc/api/embedding.md diff --git a/doc/api/embedding.md b/doc/api/embedding.md new file mode 100644 index 00000000000000..f1ab0db5a5329d --- /dev/null +++ b/doc/api/embedding.md @@ -0,0 +1,226 @@ +# C++ Embedder API + + + +Node.js provides a number of C++ APIs that can be used to execute JavaScript +in a Node.js environment from other C++ software. + +The documentation for these APIs can be found in [src/node.h][] in the Node.js +source tree. In addition to the APIs exposed by Node.js, some required concepts +are provided by the V8 embedder API. + +Because using Node.js as an embedded library is different from writing code +that is executed by Node.js, breaking changes do not follow typical Node.js +[deprecation policy][] and may occur on each semver-major release without prior +warning. + +## Example embedding application + +The following sections will provide an overview over how to use these APIs +to create an application from scratch that will perform the equivalent of +`node -e `, i.e. that will take a piece of JavaScript and run it in +a Node.js-specific environment. + +The full code can be found [in the Node.js source tree][embedtest.cc]. + +### Setting up per-process state + +Node.js requires some per-process state management in order to run: + +* Arguments parsing for Node.js [CLI options][], +* V8 per-process requirements, such as a `v8::Platform` instance. + +The following example shows how these can be set up. Some class names are from +the `node` and `v8` C++ namespaces, respectively. + +```cpp +int main(int argc, char** argv) { + std::vector args(argv, argv + argc); + std::vector exec_args; + std::vector errors; + // Parse Node.js CLI options, and print any errors that have occurred while + // trying to parse them. + int exit_code = node::InitializeNodeWithArgs(&args, &exec_args, &errors); + for (const std::string& error : errors) + fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str()); + if (exit_code != 0) { + return exit_code; + } + + // Create a v8::Platform instance. `MultiIsolatePlatform::Create()` is a way + // to create a v8::Platform instance that Node.js can use when creating + // Worker threads. When no `MultiIsolatePlatform` instance is present, + // Worker threads are disabled. + std::unique_ptr platform = + MultiIsolatePlatform::Create(4); + V8::InitializePlatform(platform.get()); + V8::Initialize(); + + // See below for the contents of this function. + int ret = RunNodeInstance(platform.get(), args, exec_args); + + V8::Dispose(); + V8::ShutdownPlatform(); + return ret; +} +``` + +### Per-instance state + +Node.js has a concept of a “Node.js instance”, that is commonly being referred +to as `node::Environment`. Each `node::Environment` is associated with: + +* Exactly one `v8::Isolate`, i.e. one JS Engine instance, +* Exactly one `uv_loop_t`, i.e. one event loop, and +* A number of `v8::Context`s, but exactly one main `v8::Context`. +* One `node::IsolateData` instance that contains information that could be + shared by multiple `node::Environment`s that use the same `v8::Isolate`. + Currently, no testing if performed for this scenario. + +In order to set up a `v8::Isolate`, an `v8::ArrayBuffer::Allocator` needs +to be provided. One possible choice is the default Node.js allocator, which +can be created through `node::ArrayBufferAllocator::Create()`. Using the Node.js +allocator allows minor performance optimizations when addons use the Node.js +C++ `Buffer` API, and is required in order to track `ArrayBuffer` memory in +[`process.memoryUsage()`][]. + +Additionally, each `v8::Isolate` that is used for a Node.js instance needs to +be registered and unregistered with the `MultiIsolatePlatform` instance, if one +is being used, in order for the platform to know which event loop to use +for tasks scheduled by the `v8::Isolate`. + +The `node::NewIsolate()` helper function creates a `v8::Isolate`, +sets it up with some Node.js-specific hooks (e.g. the Node.js error handler), +and registers it with the platform automatically. + +```cpp +int RunNodeInstance(MultiIsolatePlatform* platform, + const std::vector& args, + const std::vector& exec_args) { + int exit_code = 0; + // Set up a libuv event loop. + uv_loop_t loop; + int ret = uv_loop_init(&loop); + if (ret != 0) { + fprintf(stderr, "%s: Failed to initialize loop: %s\n", + args[0].c_str(), + uv_err_name(ret)); + return 1; + } + + std::shared_ptr allocator = + ArrayBufferAllocator::Create(); + + Isolate* isolate = NewIsolate(allocator, &loop, platform); + if (isolate == nullptr) { + fprintf(stderr, "%s: Failed to initialize V8 Isolate\n", args[0].c_str()); + return 1; + } + + { + Locker locker(isolate); + Isolate::Scope isolate_scope(isolate); + + // Create a node::IsolateData instance that will later be released using + // node::FreeIsolateData(). + std::unique_ptr isolate_data( + node::CreateIsolateData(isolate, &loop, platform, allocator.get()), + node::FreeIsolateData); + + // Set up a new v8::Context. + HandleScope handle_scope(isolate); + Local context = node::NewContext(isolate); + if (context.IsEmpty()) { + fprintf(stderr, "%s: Failed to initialize V8 Context\n", args[0].c_str()); + return 1; + } + + // The v8::Context needs to be entered when node::CreateEnvironment() and + // node::LoadEnvironment() are being called. + Context::Scope context_scope(context); + + // Create a node::Environment instance that will later be released using + // node::FreeEnvironment(). + std::unique_ptr env( + node::CreateEnvironment(isolate_data.get(), context, args, exec_args), + node::FreeEnvironment); + + // Set up the Node.js instance for execution, and run code inside of it. + // There is also a variant that takes a callback and provides it with + // the `require` and `process` objects, so that it can manually compile + // and run scripts as needed. + // The `require` function inside this script does *not* access the file + // system, and can only load built-in Node.js modules. + // `module.createRequire()` is being used to create one that is able to + // load files from the disk, and uses the standard CommonJS file loader + // instead of the internal-only `require` function. + MaybeLocal loadenv_ret = node::LoadEnvironment( + env.get(), + "const publicRequire =" + " require('module').createRequire(process.cwd() + '/');" + "globalThis.require = publicRequire;" + "require('vm').runInThisContext(process.argv[1]);"); + + if (loadenv_ret.IsEmpty()) // There has been a JS exception. + return 1; + + { + // SealHandleScope protects against handle leaks from callbacks. + SealHandleScope seal(isolate); + bool more; + do { + uv_run(&loop, UV_RUN_DEFAULT); + + // V8 tasks on background threads may end up scheduling new tasks in the + // foreground, which in turn can keep the event loop going. For example, + // WebAssembly.compile() may do so. + platform->DrainTasks(isolate); + + // If there are new tasks, continue. + more = uv_loop_alive(&loop); + if (more) continue; + + // node::EmitBeforeExit() is used to emit the 'beforeExit' event on + // the `process` object. + node::EmitBeforeExit(env.get()); + + // 'beforeExit' can also schedule new work that keeps the event loop + // running. + more = uv_loop_alive(&loop); + } while (more == true); + } + + // node::EmitExit() returns the current exit code. + exit_code = node::EmitExit(env.get()); + + // node::Stop() can be used to explicitly stop the event loop and keep + // further JavaScript from running. It can be called from any thread, + // and will act like worker.terminate() if called from another thread. + node::Stop(env.get()); + } + + // Unregister the Isolate with the platform and add a listener that is called + // when the Platform is done cleaning up any state it had associated with + // the Isolate. + bool platform_finished = false; + platform->AddIsolateFinishedCallback(isolate, [](void* data) { + *static_cast(data) = true; + }, &platform_finished); + platform->UnregisterIsolate(isolate); + isolate->Dispose(); + + // Wait until the platform has cleaned up all relevant resources. + while (!platform_finished) + uv_run(&loop, UV_RUN_ONCE); + int err = uv_loop_close(&loop); + assert(err == 0); + + return exit_code; +} +``` + +[`process.memoryUsage()`]: process.html#process_process_memoryusage +[CLI options]: cli.html +[deprecation policy]: deprecations.html +[embedtest.cc]: https://github.com/nodejs/node/blob/master/test/embedding/embedtest.cc +[src/node.h]: https://github.com/nodejs/node/blob/master/src/node.h diff --git a/doc/api/index.md b/doc/api/index.md index 841e0f3804e6f8..a6d54e1601c152 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -13,9 +13,10 @@ * [Assertion testing](assert.html) * [Async hooks](async_hooks.html) * [Buffer](buffer.html) -* [C++ addons](addons.html) -* [C/C++ addons with N-API](n-api.html) -* [Child processes](child_process.html) +* [C++ Addons](addons.html) +* [C/C++ Addons with N-API](n-api.html) +* [C++ Embedder API](embedding.html) +* [Child Processes](child_process.html) * [Cluster](cluster.html) * [Command line options](cli.html) * [Console](console.html) From cb635c2dc07e893dfe6fcf73edcc8ded9548d91d Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 12 Mar 2020 13:11:48 +0100 Subject: [PATCH 415/492] test: add extended embedder cctest Add an embedder cctest that also covers a multi-Environment situation, including worker_threads-style inspector support. Co-authored-by: Joyee Cheung Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/30467 Reviewed-By: James M Snell Reviewed-By: Gireesh Punathil --- Makefile | 4 +- test/cctest/test_environment.cc | 162 ++++++++++++++++++++++++++----- test/embedding/test-embedding.js | 33 +++++++ test/embedding/test.js | 19 ---- test/embedding/testcfg.py | 6 ++ tools/test.py | 1 + 6 files changed, 182 insertions(+), 43 deletions(-) create mode 100644 test/embedding/test-embedding.js delete mode 100644 test/embedding/test.js create mode 100644 test/embedding/testcfg.py diff --git a/Makefile b/Makefile index 35eeeb618b7097..c8731b5f4da534 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ coverage-report-js: # Runs the C++ tests using the built `cctest` executable. cctest: all @out/$(BUILDTYPE)/$@ --gtest_filter=$(GTEST_FILTER) - @out/$(BUILDTYPE)/embedtest "require('./test/embedding/test.js')" + @out/$(BUILDTYPE)/embedtest "require('./test/embedding/test-embedding.js')" .PHONY: list-gtests list-gtests: @@ -534,7 +534,7 @@ test-ci: | clear-stalled bench-addons-build build-addons build-js-native-api-tes $(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \ --mode=$(BUILDTYPE_LOWER) --flaky-tests=$(FLAKY_TESTS) \ $(TEST_CI_ARGS) $(CI_JS_SUITES) $(CI_NATIVE_SUITES) $(CI_DOC) - out/Release/embedtest 'require("./test/embedding/test.js")' + out/Release/embedtest 'require("./test/embedding/test-embedding.js")' @echo "Clean up any leftover processes, error if found." ps awwx | grep Release/node | grep -v grep | cat @PS_OUT=`ps awwx | grep Release/node | grep -v grep | awk '{print $$1}'`; \ diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 7f6dbb8d9d9cfd..b0603ce35734f3 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -266,10 +266,10 @@ TEST_F(EnvironmentTest, SetImmediateCleanup) { EXPECT_EQ(env_arg, *env); called++; }); - (*env)->SetUnrefImmediate([&](node::Environment* env_arg) { + (*env)->SetImmediate([&](node::Environment* env_arg) { EXPECT_EQ(env_arg, *env); called_unref++; - }); + }, node::CallbackFlags::kUnrefed); } EXPECT_EQ(called, 1); @@ -307,25 +307,143 @@ TEST_F(EnvironmentTest, BufferWithFreeCallbackIsDetached) { CHECK_EQ(ab->ByteLength(), 0); } -TEST_F(EnvironmentTest, SetImmediateCleanup) { - int called = 0; - int called_unref = 0; - - { - const v8::HandleScope handle_scope(isolate_); - const Argv argv; - Env env {handle_scope, argv}; - - (*env)->SetImmediate([&](node::Environment* env_arg) { - EXPECT_EQ(env_arg, *env); - called++; - }); - (*env)->SetImmediate([&](node::Environment* env_arg) { - EXPECT_EQ(env_arg, *env); - called_unref++; - }, node::CallbackFlags::kUnrefed); - } +#if HAVE_INSPECTOR +TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { + // Tests that child Environments can be created through the public API + // that are accessible by the inspector. + // This test sets a global variable in the child Environment, and reads it + // back both through the inspector and inside the child Environment, and + // makes sure that those correspond to the value that was originally set. + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env {handle_scope, argv}; - EXPECT_EQ(called, 1); - EXPECT_EQ(called_unref, 0); + v8::Local context = isolate_->GetCurrentContext(); + node::LoadEnvironment(*env, + "'use strict';\n" + "const { Worker } = require('worker_threads');\n" + "const { Session } = require('inspector');\n" + + "const session = new Session();\n" + "session.connect();\n" + "session.on('NodeWorker.attachedToWorker', (\n" + " ({ params: { workerInfo, sessionId } }) => {\n" + " session.post('NodeWorker.sendMessageToWorker', {\n" + " sessionId,\n" + " message: JSON.stringify({\n" + " id: 1,\n" + " method: 'Runtime.evaluate',\n" + " params: {\n" + " expression: 'global.variableFromParent = 42;'\n" + " }\n" + " })\n" + " });\n" + " session.on('NodeWorker.receivedMessageFromWorker',\n" + " ({ params: { message } }) => {\n" + " global.messageFromWorker = \n" + " JSON.parse(message).result.result.value;\n" + " });\n" + " }));\n" + "session.post('NodeWorker.enable', { waitForDebuggerOnStart: false });\n") + .ToLocalChecked(); + + struct ChildEnvironmentData { + node::ThreadId thread_id; + std::unique_ptr inspector_parent_handle; + node::MultiIsolatePlatform* platform; + int32_t extracted_value = -1; + uv_async_t thread_stopped_async; + }; + + ChildEnvironmentData data; + data.thread_id = node::AllocateEnvironmentThreadId(); + data.inspector_parent_handle = + GetInspectorParentHandle(*env, data.thread_id, "file:///embedded.js"); + CHECK(data.inspector_parent_handle); + data.platform = GetMultiIsolatePlatform(*env); + CHECK_NOT_NULL(data.platform); + + bool thread_stopped = false; + int err = uv_async_init( + ¤t_loop, &data.thread_stopped_async, [](uv_async_t* async) { + *static_cast(async->data) = true; + uv_close(reinterpret_cast(async), nullptr); + }); + CHECK_EQ(err, 0); + data.thread_stopped_async.data = &thread_stopped; + + uv_thread_t thread; + err = uv_thread_create(&thread, [](void* arg) { + ChildEnvironmentData* data = static_cast(arg); + std::shared_ptr aba = + node::ArrayBufferAllocator::Create(); + uv_loop_t loop; + uv_loop_init(&loop); + v8::Isolate* isolate = NewIsolate(aba.get(), &loop, data->platform); + CHECK_NOT_NULL(isolate); + + { + v8::Isolate::Scope isolate_scope(isolate); + v8::HandleScope handle_scope(isolate); + + v8::Local context = node::NewContext(isolate); + CHECK(!context.IsEmpty()); + v8::Context::Scope context_scope(context); + + node::IsolateData* isolate_data = node::CreateIsolateData( + isolate, + &loop, + data->platform); + CHECK_NOT_NULL(isolate_data); + node::Environment* environment = node::CreateEnvironment( + isolate_data, + context, + { "dummy" }, + {}, + node::EnvironmentFlags::kNoFlags, + data->thread_id); + CHECK_NOT_NULL(environment); + + v8::Local extracted_value = LoadEnvironment( + environment, + "return global.variableFromParent;", + std::move(data->inspector_parent_handle)).ToLocalChecked(); + + uv_run(&loop, UV_RUN_DEFAULT); + CHECK(extracted_value->IsInt32()); + data->extracted_value = extracted_value.As()->Value(); + + node::FreeEnvironment(environment); + node::FreeIsolateData(isolate_data); + } + + data->platform->UnregisterIsolate(isolate); + isolate->Dispose(); + uv_run(&loop, UV_RUN_DEFAULT); + CHECK_EQ(uv_loop_close(&loop), 0); + + uv_async_send(&data->thread_stopped_async); + }, &data); + CHECK_EQ(err, 0); + + bool more; + do { + uv_run(¤t_loop, UV_RUN_DEFAULT); + data.platform->DrainTasks(isolate_); + more = uv_loop_alive(¤t_loop); + } while (!thread_stopped || more); + + uv_thread_join(&thread); + + v8::Local from_inspector = + context->Global()->Get( + context, + v8::String::NewFromOneByte( + isolate_, + reinterpret_cast("messageFromWorker"), + v8::NewStringType::kNormal).ToLocalChecked()) + .ToLocalChecked(); + CHECK_EQ(data.extracted_value, 42); + CHECK_EQ(from_inspector->IntegerValue(context).FromJust(), 42); } +#endif // HAVE_INSPECTOR diff --git a/test/embedding/test-embedding.js b/test/embedding/test-embedding.js new file mode 100644 index 00000000000000..43dab0ab24ea53 --- /dev/null +++ b/test/embedding/test-embedding.js @@ -0,0 +1,33 @@ +'use strict'; +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('assert'); +const child_process = require('child_process'); +const path = require('path'); + +common.allowGlobals(global.require); +let binary = process.features.debug ? + 'out/Debug/embedtest' : 'out/Release/embedtest'; +if (common.isWindows) { + binary += '.exe'; +} +binary = path.resolve(__dirname, '..', '..', binary); + +assert.strictEqual( + child_process.spawnSync(binary, ['console.log(42)']) + .stdout.toString().trim(), + '42'); + +assert.strictEqual( + child_process.spawnSync(binary, ['throw new Error()']).status, + 1); + +assert.strictEqual( + child_process.spawnSync(binary, ['process.exitCode = 8']).status, + 8); + + +const fixturePath = JSON.stringify(fixtures.path('exit.js')); +assert.strictEqual( + child_process.spawnSync(binary, [`require(${fixturePath})`, 92]).status, + 92); diff --git a/test/embedding/test.js b/test/embedding/test.js deleted file mode 100644 index a802de1849021f..00000000000000 --- a/test/embedding/test.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; -const common = require('../common'); -const assert = require('assert'); -const child_process = require('child_process'); - -common.allowGlobals(global.require); - -assert.strictEqual( - child_process.spawnSync(process.execPath, ['console.log(42)']) - .stdout.toString().trim(), - '42'); - -assert.strictEqual( - child_process.spawnSync(process.execPath, ['throw new Error()']).status, - 1); - -assert.strictEqual( - child_process.spawnSync(process.execPath, ['process.exitCode = 8']).status, - 8); diff --git a/test/embedding/testcfg.py b/test/embedding/testcfg.py new file mode 100644 index 00000000000000..a4b90f490c670f --- /dev/null +++ b/test/embedding/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.SimpleTestConfiguration(context, root, 'embedding') diff --git a/tools/test.py b/tools/test.py index 811db910608812..532b855479b7bb 100755 --- a/tools/test.py +++ b/tools/test.py @@ -1486,6 +1486,7 @@ def PrintCrashed(code): 'addons', 'benchmark', 'doctool', + 'embedding', 'internet', 'js-native-api', 'node-api', From 61eec0c6c70e00c69d3656d944feea199227f84b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 30 Mar 2020 11:01:07 +0200 Subject: [PATCH 416/492] test: wait for message from parent in embedding cctest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’ve seen this fail a few times in CI, presumably because the inspector commmand did not reach the child thread in time. Explicitly waiting seems to solve that. Refs: https://github.com/nodejs/node/pull/30467 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32563 Reviewed-By: Colin Ihrig Reviewed-By: James M Snell --- test/cctest/test_environment.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index b0603ce35734f3..88c1c2ab953608 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -406,6 +406,7 @@ TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { v8::Local extracted_value = LoadEnvironment( environment, + "while (!global.variableFromParent) {}\n" "return global.variableFromParent;", std::move(data->inspector_parent_handle)).ToLocalChecked(); From ff0a0366f7f48fdf22a774b617e0c848798a2dd8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sat, 28 Mar 2020 02:46:41 +0100 Subject: [PATCH 417/492] embedding: provide hook for custom process.exit() behaviour Embedders may not want to terminate the process when `process.exit()` is called. This provides a hook for more flexible handling of that situation. Refs: https://github.com/nodejs/node/pull/30467#issuecomment-603689644 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32531 Reviewed-By: Colin Ihrig Reviewed-By: Gireesh Punathil Reviewed-By: James M Snell --- src/api/environment.cc | 13 +++++++++++++ src/env-inl.h | 5 +++++ src/env.cc | 9 +-------- src/env.h | 5 +++++ src/node.h | 12 ++++++++++++ src/node_worker.cc | 10 +++++++--- test/cctest/test_environment.cc | 17 +++++++++++++++++ 7 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 27eb47328c510c..cf10aa398f0744 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -713,4 +713,17 @@ ThreadId AllocateEnvironmentThreadId() { return ret; } +void DefaultProcessExitHandler(Environment* env, int exit_code) { + env->set_can_call_into_js(false); + env->stop_sub_worker_contexts(); + DisposePlatform(); + exit(exit_code); +} + + +void SetProcessExitHandler(Environment* env, + std::function&& handler) { + env->set_process_exit_handler(std::move(handler)); +} + } // namespace node diff --git a/src/env-inl.h b/src/env-inl.h index cb10b4d9554e23..0e3b2842e4d1ad 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -1221,6 +1221,11 @@ void Environment::set_main_utf16(std::unique_ptr str) { main_utf16_ = std::move(str); } +void Environment::set_process_exit_handler( + std::function&& handler) { + process_exit_handler_ = std::move(handler); +} + #define VP(PropertyName, StringValue) V(v8::Private, PropertyName) #define VY(PropertyName, StringValue) V(v8::Symbol, PropertyName) #define VS(PropertyName, StringValue) V(v8::String, PropertyName) diff --git a/src/env.cc b/src/env.cc index f0b8b521f70363..b2e4721b96db1b 100644 --- a/src/env.cc +++ b/src/env.cc @@ -987,14 +987,7 @@ void Environment::Exit(int exit_code) { StackTrace::CurrentStackTrace( isolate(), stack_trace_limit(), StackTrace::kDetailed)); } - if (is_main_thread()) { - set_can_call_into_js(false); - stop_sub_worker_contexts(); - DisposePlatform(); - exit(exit_code); - } else { - worker_context()->Exit(exit_code); - } + process_exit_handler_(this, exit_code); } void Environment::stop_sub_worker_contexts() { diff --git a/src/env.h b/src/env.h index 0566c2d26dc4f2..13c562133ce44a 100644 --- a/src/env.h +++ b/src/env.h @@ -1270,6 +1270,8 @@ class Environment : public MemoryRetainer { std::shared_ptr); inline void set_main_utf16(std::unique_ptr); + inline void set_process_exit_handler( + std::function&& handler); private: inline void ThrowError(v8::Local (*fun)(v8::Local), @@ -1428,6 +1430,9 @@ class Environment : public MemoryRetainer { ArrayBufferAllocatorList; ArrayBufferAllocatorList* keep_alive_allocators_ = nullptr; + std::function process_exit_handler_ { + DefaultProcessExitHandler }; + template void ForEachBaseObject(T&& iterator); diff --git a/src/node.h b/src/node.h index cb3ff139278ecd..74fe59097ff673 100644 --- a/src/node.h +++ b/src/node.h @@ -452,6 +452,18 @@ NODE_EXTERN v8::MaybeLocal LoadEnvironment( std::unique_ptr inspector_parent_handle = {}); NODE_EXTERN void FreeEnvironment(Environment* env); +// Set a callback that is called when process.exit() is called from JS, +// overriding the default handler. +// It receives the Environment* instance and the exit code as arguments. +// This could e.g. call Stop(env); in order to terminate execution and stop +// the event loop. +// The default handler disposes of the global V8 platform instance, if one is +// being used, and calls exit(). +NODE_EXTERN void SetProcessExitHandler( + Environment* env, + std::function&& handler); +NODE_EXTERN void DefaultProcessExitHandler(Environment* env, int exit_code); + // This may return nullptr if context is not associated with a Node instance. NODE_EXTERN Environment* GetCurrentEnvironment(v8::Local context); diff --git a/src/node_worker.cc b/src/node_worker.cc index a61bf51f1c9a02..afa4a9a52d0192 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -321,6 +321,9 @@ void Worker::Run() { if (is_stopped()) return; CHECK_NOT_NULL(env_); env_->set_env_vars(std::move(env_vars_)); + SetProcessExitHandler(env_.get(), [this](Environment*, int exit_code) { + Exit(exit_code); + }); } { Mutex::ScopedLock lock(mutex_); @@ -430,9 +433,10 @@ void Worker::JoinThread() { MakeCallback(env()->onexit_string(), arraysize(args), args); } - // We cleared all libuv handles bound to this Worker above, - // the C++ object is no longer needed for anything now. - MakeWeak(); + // If we get here, the !thread_joined_ condition at the top of the function + // implies that the thread was running. In that case, its final action will + // be to schedule a callback on the parent thread which will delete this + // object, so there's nothing more to do here. } Worker::~Worker() { diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 88c1c2ab953608..33361c12280d9d 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -448,3 +448,20 @@ TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { CHECK_EQ(from_inspector->IntegerValue(context).FromJust(), 42); } #endif // HAVE_INSPECTOR + +TEST_F(EnvironmentTest, ExitHandlerTest) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + + int callback_calls = 0; + + Env env {handle_scope, argv}; + SetProcessExitHandler(*env, [&](node::Environment* env_, int exit_code) { + EXPECT_EQ(*env, env_); + EXPECT_EQ(exit_code, 42); + callback_calls++; + node::Stop(*env); + }); + node::LoadEnvironment(*env, "process.exit(42)").ToLocalChecked(); + EXPECT_EQ(callback_calls, 1); +} From 7a8f59f1d6860d1abda45d7f2aaad0f188f71b83 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 29 Mar 2020 15:07:00 +0200 Subject: [PATCH 418/492] embedding: make Stop() stop Workers This makes sense given that terminating execution of the parent thread this way likely also is supposed to stop all running Worker threads spawned by it. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32531 Reviewed-By: Colin Ihrig Reviewed-By: Gireesh Punathil Reviewed-By: James M Snell --- src/api/environment.cc | 3 +-- src/env.cc | 3 ++- src/env.h | 2 +- src/node.cc | 2 +- src/node.h | 7 ++++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index cf10aa398f0744..e47c5d42292615 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -714,8 +714,7 @@ ThreadId AllocateEnvironmentThreadId() { } void DefaultProcessExitHandler(Environment* env, int exit_code) { - env->set_can_call_into_js(false); - env->stop_sub_worker_contexts(); + Stop(env); DisposePlatform(); exit(exit_code); } diff --git a/src/env.cc b/src/env.cc index b2e4721b96db1b..5d29deedd70ba8 100644 --- a/src/env.cc +++ b/src/env.cc @@ -523,9 +523,10 @@ void Environment::InitializeLibuv(bool start_profiler_idle_notifier) { } } -void Environment::ExitEnv() { +void Environment::Stop() { set_can_call_into_js(false); set_stopping(true); + stop_sub_worker_contexts(); isolate_->TerminateExecution(); SetImmediateThreadsafe([](Environment* env) { uv_stop(env->event_loop()); }); } diff --git a/src/env.h b/src/env.h index 13c562133ce44a..e9e760bbb8b3b0 100644 --- a/src/env.h +++ b/src/env.h @@ -913,7 +913,7 @@ class Environment : public MemoryRetainer { void RegisterHandleCleanups(); void CleanupHandles(); void Exit(int code); - void ExitEnv(); + void Stop(); // Register clean-up cb to be called on environment destruction. inline void RegisterHandleCleanup(uv_handle_t* handle, diff --git a/src/node.cc b/src/node.cc index 46e8f74cc286f7..00ae36cc0fe669 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1035,7 +1035,7 @@ int Start(int argc, char** argv) { } int Stop(Environment* env) { - env->ExitEnv(); + env->Stop(); return 0; } diff --git a/src/node.h b/src/node.h index 74fe59097ff673..eea39ed44eda0c 100644 --- a/src/node.h +++ b/src/node.h @@ -218,7 +218,8 @@ class Environment; NODE_EXTERN int Start(int argc, char* argv[]); // Tear down Node.js while it is running (there are active handles -// in the loop and / or actively executing JavaScript code). +// in the loop and / or actively executing JavaScript code). This also stops +// all Workers that may have been started earlier. NODE_EXTERN int Stop(Environment* env); // TODO(addaleax): Officially deprecate this and replace it with something @@ -457,8 +458,8 @@ NODE_EXTERN void FreeEnvironment(Environment* env); // It receives the Environment* instance and the exit code as arguments. // This could e.g. call Stop(env); in order to terminate execution and stop // the event loop. -// The default handler disposes of the global V8 platform instance, if one is -// being used, and calls exit(). +// The default handler calls Stop(), disposes of the global V8 platform +// instance, if one is being used, and calls exit(). NODE_EXTERN void SetProcessExitHandler( Environment* env, std::function&& handler); From bdf6d41c72740121cbab4180d27ee22b208528f4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sat, 21 Mar 2020 11:25:14 +0100 Subject: [PATCH 419/492] test: use InitializeNodeWithArgs in cctest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs: https://github.com/nodejs/node/commit/d7f11077f15f52a2db191d3a5bcc41581cb7361f Fixes: https://github.com/nodejs/node/issues/30257 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32406 Reviewed-By: Michaël Zasso Reviewed-By: Colin Ihrig Reviewed-By: Jiawen Geng Reviewed-By: Matheus Marchini Reviewed-By: David Carlier Reviewed-By: James M Snell --- test/cctest/node_test_fixture.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/cctest/node_test_fixture.h b/test/cctest/node_test_fixture.h index 291bc5962bed77..2f7ab06c14334f 100644 --- a/test/cctest/node_test_fixture.h +++ b/test/cctest/node_test_fixture.h @@ -73,11 +73,13 @@ class NodeTestFixture : public ::testing::Test { if (!node_initialized) { uv_os_unsetenv("NODE_OPTIONS"); node_initialized = true; - int argc = 1; - const char* argv0 = "cctest"; - int exec_argc; - const char** exec_argv; - node::Init(&argc, &argv0, &exec_argc, &exec_argv); + std::vector argv { "cctest" }; + std::vector exec_argv; + std::vector errors; + + int exitcode = node::InitializeNodeWithArgs(&argv, &exec_argv, &errors); + CHECK_EQ(exitcode, 0); + CHECK(errors.empty()); } tracing_agent = std::make_unique(); From bd71cdf1534135be01ed432d4ca3bff2237372a2 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 22 Mar 2020 11:50:20 +0100 Subject: [PATCH 420/492] test: use common.buildType in embedding test This un-breaks testing in the case of `./configure --debug-node`. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32422 Reviewed-By: Richard Lau Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca Reviewed-By: James M Snell --- test/embedding/test-embedding.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/embedding/test-embedding.js b/test/embedding/test-embedding.js index 43dab0ab24ea53..ac50f22e867de1 100644 --- a/test/embedding/test-embedding.js +++ b/test/embedding/test-embedding.js @@ -6,8 +6,7 @@ const child_process = require('child_process'); const path = require('path'); common.allowGlobals(global.require); -let binary = process.features.debug ? - 'out/Debug/embedtest' : 'out/Release/embedtest'; +let binary = `out/${common.buildType}/embedtest`; if (common.isWindows) { binary += '.exe'; } From 1066341cd90ffbad77f3f207e09ee15c3e4da7f4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 5 Apr 2020 23:13:31 +0200 Subject: [PATCH 421/492] src: initialize inspector before RunBootstrapping() This is necessary for `--inspect-brk-node` to work, and for the inspector to be aware of scripts created before bootstrapping. Fixes: https://github.com/nodejs/node/issues/32648 Refs: https://github.com/nodejs/node/pull/30467#discussion_r396879908 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32672 Reviewed-By: Gus Caplan Reviewed-By: Eugene Ostroukhov Reviewed-By: Joyee Cheung Reviewed-By: David Carlier Reviewed-By: James M Snell --- src/api/environment.cc | 56 +++++++++---------- src/node.h | 18 +++--- src/node_worker.cc | 9 +-- test/cctest/node_test_fixture.h | 12 +++- test/cctest/test_environment.cc | 11 ++-- .../test-inspector-inspect-brk-node.js | 28 ++++++++++ 6 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 test/parallel/test-inspector-inspect-brk-node.js diff --git a/src/api/environment.cc b/src/api/environment.cc index e47c5d42292615..ad980c0694a90a 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -317,6 +317,19 @@ void FreeIsolateData(IsolateData* isolate_data) { delete isolate_data; } +InspectorParentHandle::~InspectorParentHandle() {} + +// Hide the internal handle class from the public API. +#if HAVE_INSPECTOR +struct InspectorParentHandleImpl : public InspectorParentHandle { + std::unique_ptr impl; + + explicit InspectorParentHandleImpl( + std::unique_ptr&& impl) + : impl(std::move(impl)) {} +}; +#endif + Environment* CreateEnvironment(IsolateData* isolate_data, Local context, int argc, @@ -335,7 +348,8 @@ Environment* CreateEnvironment( const std::vector& args, const std::vector& exec_args, EnvironmentFlags::Flags flags, - ThreadId thread_id) { + ThreadId thread_id, + std::unique_ptr inspector_parent_handle) { Isolate* isolate = context->GetIsolate(); HandleScope handle_scope(isolate); Context::Scope context_scope(context); @@ -352,6 +366,16 @@ Environment* CreateEnvironment( env->set_abort_on_uncaught_exception(false); } +#if HAVE_INSPECTOR + if (inspector_parent_handle) { + env->InitializeInspector( + std::move(static_cast( + inspector_parent_handle.get())->impl)); + } else { + env->InitializeInspector({}); + } +#endif + if (env->RunBootstrapping().IsEmpty()) { FreeEnvironment(env); return nullptr; @@ -381,19 +405,6 @@ void FreeEnvironment(Environment* env) { delete env; } -InspectorParentHandle::~InspectorParentHandle() {} - -// Hide the internal handle class from the public API. -#if HAVE_INSPECTOR -struct InspectorParentHandleImpl : public InspectorParentHandle { - std::unique_ptr impl; - - explicit InspectorParentHandleImpl( - std::unique_ptr&& impl) - : impl(std::move(impl)) {} -}; -#endif - NODE_EXTERN std::unique_ptr GetInspectorParentHandle( Environment* env, ThreadId thread_id, @@ -417,27 +428,17 @@ void LoadEnvironment(Environment* env) { MaybeLocal LoadEnvironment( Environment* env, StartExecutionCallback cb, - std::unique_ptr inspector_parent_handle) { + std::unique_ptr removeme) { env->InitializeLibuv(per_process::v8_is_profiling); env->InitializeDiagnostics(); -#if HAVE_INSPECTOR - if (inspector_parent_handle) { - env->InitializeInspector( - std::move(static_cast( - inspector_parent_handle.get())->impl)); - } else { - env->InitializeInspector({}); - } -#endif - return StartExecution(env, cb); } MaybeLocal LoadEnvironment( Environment* env, const char* main_script_source_utf8, - std::unique_ptr inspector_parent_handle) { + std::unique_ptr removeme) { CHECK_NOT_NULL(main_script_source_utf8); return LoadEnvironment( env, @@ -462,8 +463,7 @@ MaybeLocal LoadEnvironment( env->process_object(), env->native_module_require()}; return ExecuteBootstrapper(env, name.c_str(), ¶ms, &args); - }, - std::move(inspector_parent_handle)); + }); } Environment* GetCurrentEnvironment(Local context) { diff --git a/src/node.h b/src/node.h index eea39ed44eda0c..fbe9c9e74061ec 100644 --- a/src/node.h +++ b/src/node.h @@ -400,6 +400,10 @@ enum Flags : uint64_t { }; } // namespace EnvironmentFlags +struct InspectorParentHandle { + virtual ~InspectorParentHandle(); +}; + // TODO(addaleax): Maybe move per-Environment options parsing here. // Returns nullptr when the Environment cannot be created e.g. there are // pending JavaScript exceptions. @@ -416,16 +420,14 @@ NODE_EXTERN Environment* CreateEnvironment( const std::vector& args, const std::vector& exec_args, EnvironmentFlags::Flags flags = EnvironmentFlags::kDefaultFlags, - ThreadId thread_id = {} /* allocates a thread id automatically */); + ThreadId thread_id = {} /* allocates a thread id automatically */, + std::unique_ptr inspector_parent_handle = {}); -struct InspectorParentHandle { - virtual ~InspectorParentHandle(); -}; // Returns a handle that can be passed to `LoadEnvironment()`, making the // child Environment accessible to the inspector as if it were a Node.js Worker. // `child_thread_id` can be created using `AllocateEnvironmentThreadId()` // and then later passed on to `CreateEnvironment()` to create the child -// Environment. +// Environment, together with the inspector handle. // This method should not be called while the parent Environment is active // on another thread. NODE_EXTERN std::unique_ptr GetInspectorParentHandle( @@ -443,14 +445,16 @@ using StartExecutionCallback = // TODO(addaleax): Deprecate this in favour of the MaybeLocal<> overload. NODE_EXTERN void LoadEnvironment(Environment* env); +// The `InspectorParentHandle` arguments here are ignored and not used. +// For passing `InspectorParentHandle`, use `CreateEnvironment()`. NODE_EXTERN v8::MaybeLocal LoadEnvironment( Environment* env, StartExecutionCallback cb, - std::unique_ptr inspector_parent_handle = {}); + std::unique_ptr ignored_donotuse_removeme = {}); NODE_EXTERN v8::MaybeLocal LoadEnvironment( Environment* env, const char* main_script_source_utf8, - std::unique_ptr inspector_parent_handle = {}); + std::unique_ptr ignored_donotuse_removeme = {}); NODE_EXTERN void FreeEnvironment(Environment* env); // Set a callback that is called when process.exit() is called from JS, diff --git a/src/node_worker.cc b/src/node_worker.cc index afa4a9a52d0192..bd43a9cfbab97f 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -317,7 +317,8 @@ void Worker::Run() { std::move(argv_), std::move(exec_argv_), EnvironmentFlags::kNoFlags, - thread_id_)); + thread_id_, + std::move(inspector_parent_handle_))); if (is_stopped()) return; CHECK_NOT_NULL(env_); env_->set_env_vars(std::move(env_vars_)); @@ -335,12 +336,8 @@ void Worker::Run() { { CreateEnvMessagePort(env_.get()); Debug(this, "Created message port for worker %llu", thread_id_.id); - if (LoadEnvironment(env_.get(), - StartExecutionCallback{}, - std::move(inspector_parent_handle_)) - .IsEmpty()) { + if (LoadEnvironment(env_.get(), StartExecutionCallback{}).IsEmpty()) return; - } Debug(this, "Loaded environment for worker %llu", thread_id_.id); } diff --git a/test/cctest/node_test_fixture.h b/test/cctest/node_test_fixture.h index 2f7ab06c14334f..903900ac9c94f1 100644 --- a/test/cctest/node_test_fixture.h +++ b/test/cctest/node_test_fixture.h @@ -125,7 +125,10 @@ class EnvironmentTestFixture : public NodeTestFixture { public: class Env { public: - Env(const v8::HandleScope& handle_scope, const Argv& argv) { + Env(const v8::HandleScope& handle_scope, + const Argv& argv, + node::EnvironmentFlags::Flags flags = + node::EnvironmentFlags::kDefaultFlags) { auto isolate = handle_scope.GetIsolate(); context_ = node::NewContext(isolate); CHECK(!context_.IsEmpty()); @@ -135,10 +138,13 @@ class EnvironmentTestFixture : public NodeTestFixture { &NodeTestFixture::current_loop, platform.get()); CHECK_NE(nullptr, isolate_data_); + std::vector args(*argv, *argv + 1); + std::vector exec_args(*argv, *argv + 1); environment_ = node::CreateEnvironment(isolate_data_, context_, - 1, *argv, - argv.nr_args(), *argv); + args, + exec_args, + flags); CHECK_NE(nullptr, environment_); } diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 33361c12280d9d..d0182ca8620fac 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -168,8 +168,9 @@ TEST_F(EnvironmentTest, AtExitRunsJS) { TEST_F(EnvironmentTest, MultipleEnvironmentsPerIsolate) { const v8::HandleScope handle_scope(isolate_); const Argv argv; + // Only one of the Environments can have default flags and own the inspector. Env env1 {handle_scope, argv}; - Env env2 {handle_scope, argv}; + Env env2 {handle_scope, argv, node::EnvironmentFlags::kNoFlags}; AtExit(*env1, at_exit_callback1); AtExit(*env2, at_exit_callback2); @@ -334,7 +335,7 @@ TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { " id: 1,\n" " method: 'Runtime.evaluate',\n" " params: {\n" - " expression: 'global.variableFromParent = 42;'\n" + " expression: 'globalThis.variableFromParent = 42;'\n" " }\n" " })\n" " });\n" @@ -401,14 +402,14 @@ TEST_F(EnvironmentTest, InspectorMultipleEmbeddedEnvironments) { { "dummy" }, {}, node::EnvironmentFlags::kNoFlags, - data->thread_id); + data->thread_id, + std::move(data->inspector_parent_handle)); CHECK_NOT_NULL(environment); v8::Local extracted_value = LoadEnvironment( environment, "while (!global.variableFromParent) {}\n" - "return global.variableFromParent;", - std::move(data->inspector_parent_handle)).ToLocalChecked(); + "return global.variableFromParent;").ToLocalChecked(); uv_run(&loop, UV_RUN_DEFAULT); CHECK(extracted_value->IsInt32()); diff --git a/test/parallel/test-inspector-inspect-brk-node.js b/test/parallel/test-inspector-inspect-brk-node.js new file mode 100644 index 00000000000000..caf17b956dcd19 --- /dev/null +++ b/test/parallel/test-inspector-inspect-brk-node.js @@ -0,0 +1,28 @@ +'use strict'; +const common = require('../common'); + +// Regression test for https://github.com/nodejs/node/issues/32648 + +common.skipIfInspectorDisabled(); + +const { NodeInstance } = require('../common/inspector-helper.js'); + +async function runTest() { + const child = new NodeInstance(['--inspect-brk-node=0', '-p', '42']); + const session = await child.connectInspectorSession(); + await session.send({ method: 'Runtime.enable' }); + await session.send({ method: 'Debugger.enable' }); + await session.send({ method: 'Runtime.runIfWaitingForDebugger' }); + await session.waitForNotification((notification) => { + // The main assertion here is that we do hit the loader script first. + return notification.method === 'Debugger.scriptParsed' && + notification.params.url === 'internal/bootstrap/loaders.js'; + }); + + await session.waitForNotification('Debugger.paused'); + await session.send({ method: 'Debugger.resume' }); + await session.waitForNotification('Debugger.paused'); + await session.runToCompletion(); +} + +runTest().then(common.mustCall()); From 18ecaebdbb5446107ebcf56254f61588dacde45d Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 27 Apr 2020 03:36:23 +0200 Subject: [PATCH 422/492] worker: unify custom error creation Mostly, this introduces a pattern that makes sure that if a custom error is reported, `stopped_` will be set to `true` correctly in every cast, which was previously missing for the `NewContext().IsEmpty()` case (which led to a hard crash from the `Worker` destructor). This also leaves TODO comments for a few cases in which `ERR_WORKER_OUT_OF_MEMORY` was not used in accordance with the documentation for that error code (or according to its intention). Fixing that is semver-major. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/33084 Reviewed-By: Gireesh Punathil Reviewed-By: Colin Ihrig Reviewed-By: James M Snell --- src/node_worker.cc | 30 +++++++++++++++++------------- src/node_worker.h | 7 +++++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/node_worker.cc b/src/node_worker.cc index bd43a9cfbab97f..db809d12f81310 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -136,9 +136,7 @@ class WorkerThreadData { if (ret != 0) { char err_buf[128]; uv_err_name_r(ret, err_buf, sizeof(err_buf)); - w->custom_error_ = "ERR_WORKER_INIT_FAILED"; - w->custom_error_str_ = err_buf; - w->stopped_ = true; + w->Exit(1, "ERR_WORKER_INIT_FAILED", err_buf); return; } loop_init_failed_ = false; @@ -152,9 +150,9 @@ class WorkerThreadData { Isolate* isolate = Isolate::Allocate(); if (isolate == nullptr) { - w->custom_error_ = "ERR_WORKER_OUT_OF_MEMORY"; - w->custom_error_str_ = "Failed to create new Isolate"; - w->stopped_ = true; + // TODO(addaleax): This should be ERR_WORKER_INIT_FAILED, + // ERR_WORKER_OUT_OF_MEMORY is for reaching the per-Worker heap limit. + w->Exit(1, "ERR_WORKER_OUT_OF_MEMORY", "Failed to create new Isolate"); return; } @@ -237,9 +235,7 @@ class WorkerThreadData { size_t Worker::NearHeapLimit(void* data, size_t current_heap_limit, size_t initial_heap_limit) { Worker* worker = static_cast(data); - worker->custom_error_ = "ERR_WORKER_OUT_OF_MEMORY"; - worker->custom_error_str_ = "JS heap out of memory"; - worker->Exit(1); + worker->Exit(1, "ERR_WORKER_OUT_OF_MEMORY", "JS heap out of memory"); // Give the current GC some extra leeway to let it finish rather than // crash hard. We are not going to perform further allocations anyway. constexpr size_t kExtraHeapAllowance = 16 * 1024 * 1024; @@ -301,8 +297,9 @@ void Worker::Run() { TryCatch try_catch(isolate_); context = NewContext(isolate_); if (context.IsEmpty()) { - custom_error_ = "ERR_WORKER_OUT_OF_MEMORY"; - custom_error_str_ = "Failed to create new Context"; + // TODO(addaleax): This should be ERR_WORKER_INIT_FAILED, + // ERR_WORKER_OUT_OF_MEMORY is for reaching the per-Worker heap limit. + Exit(1, "ERR_WORKER_OUT_OF_MEMORY", "Failed to create new Context"); return; } } @@ -685,9 +682,16 @@ Local Worker::GetResourceLimits(Isolate* isolate) const { return Float64Array::New(ab, 0, kTotalResourceLimitCount); } -void Worker::Exit(int code) { +void Worker::Exit(int code, const char* error_code, const char* error_message) { Mutex::ScopedLock lock(mutex_); - Debug(this, "Worker %llu called Exit(%d)", thread_id_.id, code); + Debug(this, "Worker %llu called Exit(%d, %s, %s)", + thread_id_.id, code, error_code, error_message); + + if (error_code != nullptr) { + custom_error_ = error_code; + custom_error_str_ = error_message; + } + if (env_ != nullptr) { exit_code_ = code; Stop(env_); diff --git a/src/node_worker.h b/src/node_worker.h index a7d02914826563..069bd190fca5f7 100644 --- a/src/node_worker.h +++ b/src/node_worker.h @@ -34,8 +34,11 @@ class Worker : public AsyncWrap { void Run(); // Forcibly exit the thread with a specified exit code. This may be called - // from any thread. - void Exit(int code); + // from any thread. `error_code` and `error_message` can be used to create + // a custom `'error'` event before emitting `'exit'`. + void Exit(int code, + const char* error_code = nullptr, + const char* error_message = nullptr); // Wait for the worker thread to stop (in a blocking manner). void JoinThread(); From 4513b6a3df1cf42f5f6fbd9f33b5709c89c2c774 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 22 Mar 2020 21:07:21 +0100 Subject: [PATCH 423/492] src: make `Environment::interrupt_data_` atomic Otherwise this potentially leads to race conditions when used in a multi-threaded environment (i.e. when used for its very purpose). Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32523 Reviewed-By: Matheus Marchini Reviewed-By: James M Snell --- src/env.cc | 24 ++++++++++++++++++------ src/env.h | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/env.cc b/src/env.cc index 5d29deedd70ba8..4f1ffbd7400f43 100644 --- a/src/env.cc +++ b/src/env.cc @@ -412,7 +412,8 @@ Environment::Environment(IsolateData* isolate_data, } Environment::~Environment() { - if (interrupt_data_ != nullptr) *interrupt_data_ = nullptr; + if (Environment** interrupt_data = interrupt_data_.load()) + *interrupt_data = nullptr; // FreeEnvironment() should have set this. CHECK(is_stopping()); @@ -737,12 +738,23 @@ void Environment::RunAndClearNativeImmediates(bool only_refed) { } void Environment::RequestInterruptFromV8() { - if (interrupt_data_ != nullptr) return; // Already scheduled. - // The Isolate may outlive the Environment, so some logic to handle the // situation in which the Environment is destroyed before the handler runs // is required. - interrupt_data_ = new Environment*(this); + + // We allocate a new pointer to a pointer to this Environment instance, and + // try to set it as interrupt_data_. If interrupt_data_ was already set, then + // callbacks are already scheduled to run and we can delete our own pointer + // and just return. If it was nullptr previously, the Environment** is stored; + // ~Environment sets the Environment* contained in it to nullptr, so that + // the callback can check whether ~Environment has already run and it is thus + // not safe to access the Environment instance itself. + Environment** interrupt_data = new Environment*(this); + Environment** dummy = nullptr; + if (!interrupt_data_.compare_exchange_strong(dummy, interrupt_data)) { + delete interrupt_data; + return; // Already scheduled. + } isolate()->RequestInterrupt([](Isolate* isolate, void* data) { std::unique_ptr env_ptr { static_cast(data) }; @@ -753,9 +765,9 @@ void Environment::RequestInterruptFromV8() { // handled during cleanup. return; } - env->interrupt_data_ = nullptr; + env->interrupt_data_.store(nullptr); env->RunAndClearInterrupts(); - }, interrupt_data_); + }, interrupt_data); } void Environment::ScheduleTimer(int64_t duration_ms) { diff --git a/src/env.h b/src/env.h index e9e760bbb8b3b0..98a3172075ce6e 100644 --- a/src/env.h +++ b/src/env.h @@ -1412,7 +1412,7 @@ class Environment : public MemoryRetainer { void RunAndClearNativeImmediates(bool only_refed = false); void RunAndClearInterrupts(); - Environment** interrupt_data_ = nullptr; + std::atomic interrupt_data_ {nullptr}; void RequestInterruptFromV8(); static void CheckImmediate(uv_check_t* handle); From 4704e586dcc2b759b6bdc816f7f723d41f5ac49c Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Fri, 27 Mar 2020 21:38:39 +0100 Subject: [PATCH 424/492] src: fix cleanup hook removal for InspectorTimer Fix this to account for the fact that `Stop()` may already have been called from a cleanup hook when the `inspector::Agent` is deleted along with the `Environment`, at which point cleanup hooks are no longer available. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32523 Reviewed-By: Matheus Marchini Reviewed-By: James M Snell --- src/inspector_agent.cc | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index c53e3af1d55f65..b9da4e4d5a6100 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -355,32 +355,26 @@ class InspectorTimer { int64_t interval_ms = 1000 * interval_s; uv_timer_start(&timer_, OnTimer, interval_ms, interval_ms); timer_.data = this; - - env->AddCleanupHook(CleanupHook, this); } InspectorTimer(const InspectorTimer&) = delete; void Stop() { - env_->RemoveCleanupHook(CleanupHook, this); + if (timer_.data == nullptr) return; - if (timer_.data == this) { - timer_.data = nullptr; - uv_timer_stop(&timer_); - env_->CloseHandle(reinterpret_cast(&timer_), TimerClosedCb); - } + timer_.data = nullptr; + uv_timer_stop(&timer_); + env_->CloseHandle(reinterpret_cast(&timer_), TimerClosedCb); } + inline Environment* env() const { return env_; } + private: static void OnTimer(uv_timer_t* uvtimer) { InspectorTimer* timer = node::ContainerOf(&InspectorTimer::timer_, uvtimer); timer->callback_(timer->data_); } - static void CleanupHook(void* data) { - static_cast(data)->Stop(); - } - static void TimerClosedCb(uv_handle_t* uvtimer) { std::unique_ptr timer( node::ContainerOf(&InspectorTimer::timer_, @@ -403,16 +397,29 @@ class InspectorTimerHandle { InspectorTimerHandle(Environment* env, double interval_s, V8InspectorClient::TimerCallback callback, void* data) { timer_ = new InspectorTimer(env, interval_s, callback, data); + + env->AddCleanupHook(CleanupHook, this); } InspectorTimerHandle(const InspectorTimerHandle&) = delete; ~InspectorTimerHandle() { - CHECK_NOT_NULL(timer_); - timer_->Stop(); - timer_ = nullptr; + Stop(); } + private: + void Stop() { + if (timer_ != nullptr) { + timer_->env()->RemoveCleanupHook(CleanupHook, this); + timer_->Stop(); + } + timer_ = nullptr; + } + + static void CleanupHook(void* data) { + static_cast(data)->Stop(); + } + InspectorTimer* timer_; }; @@ -735,8 +742,9 @@ class NodeInspectorClient : public V8InspectorClient { bool is_main_; bool running_nested_loop_ = false; std::unique_ptr client_; - std::unordered_map> channels_; + // Note: ~ChannelImpl may access timers_ so timers_ has to come first. std::unordered_map timers_; + std::unordered_map> channels_; int next_session_id_ = 1; bool waiting_for_resume_ = false; bool waiting_for_frontend_ = false; From 2e4536e701fd7d7bc2a9a249fe917636936595bb Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 22 Mar 2020 19:03:24 +0100 Subject: [PATCH 425/492] src: use env->RequestInterrupt() for inspector io thread start This cleans up `Agent::RequestIoThreadStart()` significantly. Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32523 Reviewed-By: Matheus Marchini Reviewed-By: James M Snell --- src/inspector_agent.cc | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index b9da4e4d5a6100..7d665aa5d1381f 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -46,7 +46,6 @@ using v8::Local; using v8::Message; using v8::Object; using v8::Task; -using v8::TaskRunner; using v8::Value; using v8_inspector::StringBuffer; using v8_inspector::StringView; @@ -61,18 +60,6 @@ static std::atomic_bool start_io_thread_async_initialized { false }; // Protects the Agent* stored in start_io_thread_async.data. static Mutex start_io_thread_async_mutex; -class StartIoTask : public Task { - public: - explicit StartIoTask(Agent* agent) : agent(agent) {} - - void Run() override { - agent->StartIoThread(); - } - - private: - Agent* agent; -}; - std::unique_ptr ToProtocolString(Isolate* isolate, Local value) { TwoByteValue buffer(isolate, value); @@ -84,10 +71,6 @@ void StartIoThreadAsyncCallback(uv_async_t* handle) { static_cast(handle->data)->StartIoThread(); } -void StartIoInterrupt(Isolate* isolate, void* agent) { - static_cast(agent)->StartIoThread(); -} - #ifdef __POSIX__ static void StartIoThreadWakeup(int signo) { @@ -981,12 +964,10 @@ void Agent::RequestIoThreadStart() { // for IO events) CHECK(start_io_thread_async_initialized); uv_async_send(&start_io_thread_async); - Isolate* isolate = parent_env_->isolate(); - v8::Platform* platform = parent_env_->isolate_data()->platform(); - std::shared_ptr taskrunner = - platform->GetForegroundTaskRunner(isolate); - taskrunner->PostTask(std::make_unique(this)); - isolate->RequestInterrupt(StartIoInterrupt, this); + parent_env_->RequestInterrupt([this](Environment*) { + StartIoThread(); + }); + CHECK(start_io_thread_async_initialized); uv_async_send(&start_io_thread_async); } From 29620c41fbcd46512608ca5e0fb5cc9fd2f29794 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sun, 22 Mar 2020 19:04:57 +0100 Subject: [PATCH 426/492] src: use env->RequestInterrupt() for inspector MainThreadInterface This simplifies the code significantly, and removes the dependency of the inspector code on the availability of a `MultiIsolatePlatform` (by removing the dependency on a platform altogether). It also fixes a memory leak that occurs when `RequestInterrupt()` is used, but the interrupt handler is never called before the Isolate is destroyed. One downside is that this leads to a slight change in timing, because inspector messages are not dispatched in a re-entrant way. This means having to harden one test to account for that possibility by making sure that the stack is always clear through a `setImmediate()`. This does not affect the assertion made by the test, which is that messages will not be delivered synchronously while other code is executing. https://github.com/nodejs/node/issues/32415 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32523 Reviewed-By: Matheus Marchini Reviewed-By: James M Snell --- src/env.h | 5 ++- src/inspector/main_thread_interface.cc | 41 ++++--------------- src/inspector/main_thread_interface.h | 5 +-- src/inspector_agent.cc | 6 +-- src/inspector_agent.h | 2 + src/inspector_js_api.cc | 2 +- .../test-inspector-connect-main-thread.js | 6 +++ 7 files changed, 23 insertions(+), 44 deletions(-) diff --git a/src/env.h b/src/env.h index 98a3172075ce6e..f947f0bf4301fc 100644 --- a/src/env.h +++ b/src/env.h @@ -1273,6 +1273,9 @@ class Environment : public MemoryRetainer { inline void set_process_exit_handler( std::function&& handler); + void RunAndClearNativeImmediates(bool only_refed = false); + void RunAndClearInterrupts(); + private: inline void ThrowError(v8::Local (*fun)(v8::Local), const char* errmsg); @@ -1410,8 +1413,6 @@ class Environment : public MemoryRetainer { // yet or already have been destroyed. bool task_queues_async_initialized_ = false; - void RunAndClearNativeImmediates(bool only_refed = false); - void RunAndClearInterrupts(); std::atomic interrupt_data_ {nullptr}; void RequestInterruptFromV8(); static void CheckImmediate(uv_check_t* handle); diff --git a/src/inspector/main_thread_interface.cc b/src/inspector/main_thread_interface.cc index 48a964d564fec2..a15cd52d239e40 100644 --- a/src/inspector/main_thread_interface.cc +++ b/src/inspector/main_thread_interface.cc @@ -1,5 +1,6 @@ #include "main_thread_interface.h" +#include "env-inl.h" #include "node_mutex.h" #include "v8-inspector.h" #include "util-inl.h" @@ -85,19 +86,6 @@ class CallRequest : public Request { Fn fn_; }; -class DispatchMessagesTask : public v8::Task { - public: - explicit DispatchMessagesTask(std::weak_ptr thread) - : thread_(thread) {} - - void Run() override { - if (auto thread = thread_.lock()) thread->DispatchMessages(); - } - - private: - std::weak_ptr thread_; -}; - template class AnotherThreadObjectReference { public: @@ -212,12 +200,7 @@ class ThreadSafeDelegate : public InspectorSessionDelegate { } // namespace -MainThreadInterface::MainThreadInterface(Agent* agent, uv_loop_t* loop, - v8::Isolate* isolate, - v8::Platform* platform) - : agent_(agent), isolate_(isolate), - platform_(platform) { -} +MainThreadInterface::MainThreadInterface(Agent* agent) : agent_(agent) {} MainThreadInterface::~MainThreadInterface() { if (handle_) @@ -225,23 +208,15 @@ MainThreadInterface::~MainThreadInterface() { } void MainThreadInterface::Post(std::unique_ptr request) { + CHECK_NOT_NULL(agent_); Mutex::ScopedLock scoped_lock(requests_lock_); bool needs_notify = requests_.empty(); requests_.push_back(std::move(request)); if (needs_notify) { - if (isolate_ != nullptr && platform_ != nullptr) { - std::shared_ptr taskrunner = - platform_->GetForegroundTaskRunner(isolate_); - std::weak_ptr* interface_ptr = - new std::weak_ptr(shared_from_this()); - taskrunner->PostTask( - std::make_unique(*interface_ptr)); - isolate_->RequestInterrupt([](v8::Isolate* isolate, void* opaque) { - std::unique_ptr> interface_ptr { - static_cast*>(opaque) }; - if (auto iface = interface_ptr->lock()) iface->DispatchMessages(); - }, static_cast(interface_ptr)); - } + std::weak_ptr weak_self {shared_from_this()}; + agent_->env()->RequestInterrupt([weak_self](Environment*) { + if (auto iface = weak_self.lock()) iface->DispatchMessages(); + }); } incoming_message_cond_.Broadcast(scoped_lock); } @@ -274,7 +249,7 @@ void MainThreadInterface::DispatchMessages() { std::swap(dispatching_message_queue_.front(), task); dispatching_message_queue_.pop_front(); - v8::SealHandleScope seal_handle_scope(isolate_); + v8::SealHandleScope seal_handle_scope(agent_->env()->isolate()); task->Call(this); } } while (had_messages); diff --git a/src/inspector/main_thread_interface.h b/src/inspector/main_thread_interface.h index 78337a25e43808..3ec5b3db3e1600 100644 --- a/src/inspector/main_thread_interface.h +++ b/src/inspector/main_thread_interface.h @@ -72,8 +72,7 @@ class MainThreadHandle : public std::enable_shared_from_this { class MainThreadInterface : public std::enable_shared_from_this { public: - MainThreadInterface(Agent* agent, uv_loop_t*, v8::Isolate* isolate, - v8::Platform* platform); + explicit MainThreadInterface(Agent* agent); ~MainThreadInterface(); void DispatchMessages(); @@ -98,8 +97,6 @@ class MainThreadInterface : ConditionVariable incoming_message_cond_; // Used from any thread Agent* const agent_; - v8::Isolate* const isolate_; - v8::Platform* const platform_; std::shared_ptr handle_; std::unordered_map> managed_objects_; }; diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index 7d665aa5d1381f..26101fe0e13b5e 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -660,8 +660,7 @@ class NodeInspectorClient : public V8InspectorClient { std::shared_ptr getThreadHandle() { if (!interface_) { interface_ = std::make_shared( - env_->inspector_agent(), env_->event_loop(), env_->isolate(), - env_->isolate_data()->platform()); + env_->inspector_agent()); } return interface_->GetHandle(); } @@ -697,10 +696,9 @@ class NodeInspectorClient : public V8InspectorClient { running_nested_loop_ = true; - MultiIsolatePlatform* platform = env_->isolate_data()->platform(); while (shouldRunMessageLoop()) { if (interface_) interface_->WaitForFrontendEvent(); - while (platform->FlushForegroundTasks(env_->isolate())) {} + env_->RunAndClearInterrupts(); } running_nested_loop_ = false; } diff --git a/src/inspector_agent.h b/src/inspector_agent.h index 089077370db049..efd090c49b4311 100644 --- a/src/inspector_agent.h +++ b/src/inspector_agent.h @@ -116,6 +116,8 @@ class Agent { // Interface for interacting with inspectors in worker threads std::shared_ptr GetWorkerManager(); + inline Environment* env() const { return parent_env_; } + private: void ToggleAsyncHook(v8::Isolate* isolate, const v8::Global& fn); diff --git a/src/inspector_js_api.cc b/src/inspector_js_api.cc index acc767cb8745bb..8616575e066121 100644 --- a/src/inspector_js_api.cc +++ b/src/inspector_js_api.cc @@ -83,7 +83,7 @@ class JSBindingsConnection : public AsyncWrap { private: Environment* env_; - JSBindingsConnection* connection_; + BaseObjectPtr connection_; }; JSBindingsConnection(Environment* env, diff --git a/test/parallel/test-inspector-connect-main-thread.js b/test/parallel/test-inspector-connect-main-thread.js index bb21a54e90f1d1..be34981cd0c5be 100644 --- a/test/parallel/test-inspector-connect-main-thread.js +++ b/test/parallel/test-inspector-connect-main-thread.js @@ -76,6 +76,12 @@ function doConsoleLog(arrayBuffer) { // and do not interrupt the main code. Interrupting the code flow // can lead to unexpected behaviors. async function ensureListenerDoesNotInterrupt(session) { + // Make sure that the following code is not affected by the fact that it may + // run inside an inspector message callback, during which other inspector + // message callbacks (such as the one triggered by doConsoleLog()) would + // not be processed. + await new Promise(setImmediate); + const currentTime = Date.now(); let consoleLogHappened = false; session.once('Runtime.consoleAPICalled', From 147440510f559e0d0d24528354146e9f89cbe5f3 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Sat, 28 Mar 2020 01:14:06 +0100 Subject: [PATCH 427/492] src: flush V8 interrupts from Environment dtor This avoids an edge-case memory leak. Refs: https://github.com/nodejs/node/pull/32523#discussion_r399554491 Backport-PR-URL: https://github.com/nodejs/node/pull/35241 PR-URL: https://github.com/nodejs/node/pull/32523 Reviewed-By: Matheus Marchini Reviewed-By: James M Snell --- src/env.cc | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/env.cc b/src/env.cc index 4f1ffbd7400f43..703a2a15758f95 100644 --- a/src/env.cc +++ b/src/env.cc @@ -41,11 +41,13 @@ using v8::NewStringType; using v8::Number; using v8::Object; using v8::Private; +using v8::Script; using v8::SnapshotCreator; using v8::StackTrace; using v8::String; using v8::Symbol; using v8::TracingController; +using v8::TryCatch; using v8::Undefined; using v8::Value; using worker::Worker; @@ -412,11 +414,30 @@ Environment::Environment(IsolateData* isolate_data, } Environment::~Environment() { - if (Environment** interrupt_data = interrupt_data_.load()) + if (Environment** interrupt_data = interrupt_data_.load()) { + // There are pending RequestInterrupt() callbacks. Tell them not to run, + // then force V8 to run interrupts by compiling and running an empty script + // so as not to leak memory. *interrupt_data = nullptr; - // FreeEnvironment() should have set this. - CHECK(is_stopping()); + Isolate::AllowJavascriptExecutionScope allow_js_here(isolate()); + HandleScope handle_scope(isolate()); + TryCatch try_catch(isolate()); + Context::Scope context_scope(context()); + +#ifdef DEBUG + bool consistency_check = false; + isolate()->RequestInterrupt([](Isolate*, void* data) { + *static_cast(data) = true; + }, &consistency_check); +#endif + + Local
Constant
ctrl + w or ctrl + backspaceDelete backwards to a word boundaryDelete backward to a word boundary ctrl + backspace Doesn't work on Linux, Mac and Windows