diff --git a/.github/workflows/nodejs-test.yml b/.github/workflows/nodejs-test.yml index e7d1e23..8e6c923 100644 --- a/.github/workflows/nodejs-test.yml +++ b/.github/workflows/nodejs-test.yml @@ -25,5 +25,22 @@ jobs: npm test env: CI: true - - name: JSR Publish Test + - name: JSR Publish Test run: npm run test:jsr + + emfile_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 22.x + uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: npm install and build + run: | + npm install + npm run build --if-present + env: + CI: true + - name: Run EMFILE test + run: npm run test:emfile diff --git a/.gitignore b/.gitignore index 929aa4d..50608a4 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ tests/fixtures/typescript-project/index.js # file used to generate env.d.ts dist/env.js + +tmp diff --git a/package.json b/package.json index dcdf997..e61c32d 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "test:unit": "mocha tests/retrier.test.js", "test:build": "node tests/pkg.test.cjs && node tests/pkg.test.mjs", "test:jsr": "npx jsr@latest publish --dry-run", + "test:emfile": "node tools/check-emfile-handling.js", "test": "npm run test:unit && npm run test:build" }, "repository": { diff --git a/src/retrier.js b/src/retrier.js index 7b8c172..1a66a8a 100644 --- a/src/retrier.js +++ b/src/retrier.js @@ -56,7 +56,7 @@ function isTimeToRetry(task, maxDelay) { * @returns {boolean} true if it is time to bail, false otherwise. */ function isTimeToBail(task, timeout) { - return Date.now() - task.timestamp > timeout; + return task.age > timeout; } @@ -64,6 +64,13 @@ function isTimeToBail(task, timeout) { * A class to represent a task in the retry queue. */ class RetryTask { + + /** + * The unique ID for the task. + * @type {string} + */ + id = Math.random().toString(36).slice(2); + /** * The function to call. * @type {Function} @@ -124,6 +131,14 @@ class RetryTask { this.signal = signal; } + /** + * Gets the age of the task. + * @returns {number} The age of the task in milliseconds. + * @readonly + */ + get age() { + return Date.now() - this.timestamp; + } } //----------------------------------------------------------------------------- @@ -134,6 +149,7 @@ class RetryTask { * A class that manages a queue of retry jobs. */ export class Retrier { + /** * Represents the queue for processing tasks. * @type {Array} @@ -240,18 +256,21 @@ export class Retrier { if (!task) { return; } + const processAgain = () => { + this.#timerId = setTimeout(() => this.#processQueue(), 0); + }; // if it's time to bail, then bail if (isTimeToBail(task, this.#timeout)) { task.reject(task.error); - this.#processQueue(); + processAgain(); return; } // if it's not time to retry, then wait and try again if (!isTimeToRetry(task, this.#maxDelay)) { - this.#queue.unshift(task); - this.#timerId = setTimeout(() => this.#processQueue(), 0); + this.#queue.push(task); + processAgain(); return; } diff --git a/tools/check-emfile-handling.js b/tools/check-emfile-handling.js new file mode 100644 index 0000000..e4fab15 --- /dev/null +++ b/tools/check-emfile-handling.js @@ -0,0 +1,131 @@ +/** + * @fileoverview A utility to test that ESLint doesn't crash with EMFILE/ENFILE errors. + * @author Nicholas C. Zakas + */ + +//------------------------------------------------------------------------------ +// Imports +//------------------------------------------------------------------------------ + +import fs from "node:fs"; +import { readFile } from "node:fs/promises"; +import os from "node:os"; +import { execSync } from "node:child_process"; +import { Retrier } from "../src/retrier.js"; + +//------------------------------------------------------------------------------ +// Helpers +//------------------------------------------------------------------------------ + +const OUTPUT_DIRECTORY = "tmp/emfile-check"; + +/* + * Every operating system has a different limit for the number of files that can + * be opened at once. This number is meant to be larger than the default limit + * on most systems. + * + * Linux systems typically start at a count of 1024 and may be increased to 4096. + * MacOS Sonoma v14.4 has a limit of 10496. + * Windows has no hard limit but may be limited by available memory. + */ +const DEFAULT_FILE_COUNT = 15000; +let FILE_COUNT = DEFAULT_FILE_COUNT; + +// if the platform isn't windows, get the ulimit to see what the actual limit is +if (os.platform() !== "win32") { + try { + FILE_COUNT = parseInt(execSync("ulimit -n").toString().trim(), 10) + 1; + + console.log(`Detected Linux file limit of ${FILE_COUNT}.`); + + // if we're on a Mac, make sure the limit isn't high enough to cause a call stack error + if (os.platform() === "darwin") { + FILE_COUNT = Math.min(FILE_COUNT, 100000); + } + } catch { + + // ignore error and use default + } +} + +/** + * Generates files in a directory. + * @returns {void} + */ +function generateFiles() { + + fs.mkdirSync(OUTPUT_DIRECTORY, { recursive: true }); + + for (let i = 0; i < FILE_COUNT; i++) { + const fileName = `file_${i}.js`; + const fileContent = `// This is file ${i}`; + + fs.writeFileSync(`${OUTPUT_DIRECTORY}/${fileName}`, fileContent); + } + +} + +/** + * Generates an EMFILE error by reading all files in the output directory. + * @returns {Promise} A promise that resolves with the contents of all files. + */ +function generateEmFileError() { + return Promise.all( + Array.from({ length: FILE_COUNT }, (_, i) => { + const fileName = `file_${i}.js`; + + return readFile(`${OUTPUT_DIRECTORY}/${fileName}`); + }) + ); +} + +/** + * Generates an EMFILE error by reading all files in the output directory with retries. + * @returns {Promise} A promise that resolves with the contents of all files. + */ +function generateEmFileErrorWithRetry() { + const retrier = new Retrier(error => error.code === "EMFILE" || error.code === "ENFILE"); + + return Promise.all( + Array.from({ length: FILE_COUNT }, (_, i) => { + const fileName = `file_${i}.js`; + + return retrier.retry(() => readFile(`${OUTPUT_DIRECTORY}/${fileName}`)); + }) + ); +} + +//------------------------------------------------------------------------------ +// Main +//------------------------------------------------------------------------------ + +console.log(`Generating ${FILE_COUNT} files in ${OUTPUT_DIRECTORY}...`); +generateFiles(); + +console.log("Checking that this number of files would cause an EMFILE error..."); +generateEmFileError() + .then(() => { + throw new Error("EMFILE error not encountered."); + }) + .catch(error => { + if (error.code === "EMFILE") { + console.log("✅ EMFILE error encountered:", error.message); + } else if (error.code === "ENFILE") { + console.log("✅ ENFILE error encountered:", error.message); + } else { + console.error("❌ Unexpected error encountered:", error.message); + throw error; + } + }).then(() => { + + console.log("Running with retry..."); + return generateEmFileErrorWithRetry() + .then(() => { + console.log("✅ No errors encountered with retry."); + }) + .catch(error => { + console.error("❌ Unexpected error encountered with retry:", error.message); + throw error; + }); + + });