From 04a89f959c885a86d82ca07f759977d2164349e7 Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Wed, 28 Feb 2024 10:37:29 +0200
Subject: [PATCH] fs: expose glob and globSync

---
 doc/api/fs.md                                 |  59 +++++
 lib/fs.js                                     |  20 ++
 lib/internal/fs/glob.js                       | 244 +++++++++++++++++-
 lib/internal/fs/promises.js                   |  20 +-
 test/parallel/test-fs-glob.mjs                |  35 ++-
 test/parallel/test-permission-fs-supported.js |   1 +
 6 files changed, 355 insertions(+), 24 deletions(-)

diff --git a/doc/api/fs.md b/doc/api/fs.md
index 4bd7da6de39030..64e106bb12c48b 100644
--- a/doc/api/fs.md
+++ b/doc/api/fs.md
@@ -1069,6 +1069,38 @@ including subdirectories and files.
 When copying a directory to another directory, globs are not supported and
 behavior is similar to `cp dir1/ dir2/`.
 
+### `fsPromises.glob(patten[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+* `patten` {string|string\[]}
+* `options` {Object}
+  * `cwd` {string} working directory. **Default:** `process.cwd()`
+  * `exclude` {Function} Function to filter out files/directories. Return
+    `true` to exclude the item, `false` to include it. **Default:** `undefined`.
+* Returns: {AsyncIterator} An AsyncIterator that yields the paths of files
+  that match the pattern.
+
+```mjs
+import { glob } from 'node:fs/promises';
+
+for await (const entry of glob('**/*.js'))
+  console.log(entry);
+```
+
+```cjs
+const { glob } = require('node:fs/promises');
+
+(async () => {
+  for await (const entry of glob('**/*.js'))
+    console.log(entry);
+})();
+```
+
 ### `fsPromises.lchmod(path, mode)`
 
 <!-- YAML
@@ -5529,6 +5561,33 @@ changes:
 
 Synchronous version of [`fs.futimes()`][]. Returns `undefined`.
 
+### `fs.globSync(patten[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+* `patten` {string|string\[]}
+* `options` {Object}
+  * `cwd` {string} working directory. **Default:** `process.cwd()`
+  * `exclude` {Function} Function to filter out files/directories. Return
+    `true` to exclude the item, `false` to include it. **Default:** `undefined`.
+* Returns: {string\[]} paths of files that match the pattern.
+
+```mjs
+import { globSync } from 'node:fs';
+
+console.log(globSync('**/*.js'));
+```
+
+```cjs
+const { globSync } = require('node:fs');
+
+console.log(globSync('**/*.js'));
+```
+
 ### `fs.lchmodSync(path, mode)`
 
 <!-- YAML
diff --git a/lib/fs.js b/lib/fs.js
index 22688f631daf88..b3e75b75975234 100644
--- a/lib/fs.js
+++ b/lib/fs.js
@@ -86,6 +86,8 @@ const {
 const { toPathIfFileURL } = require('internal/url');
 const {
   customPromisifyArgs: kCustomPromisifyArgsSymbol,
+  emitExperimentalWarning,
+  getLazy,
   kEmptyObject,
   promisify: {
     custom: kCustomPromisifiedSymbol,
@@ -142,6 +144,7 @@ const {
   validateInteger,
   validateObject,
   validateString,
+  validateStringArray,
   kValidateObjectAllowNullable,
 } = require('internal/validators');
 
@@ -3102,6 +3105,22 @@ function createWriteStream(path, options) {
   return new WriteStream(path, options);
 }
 
+const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
+
+function globSync(pattern, options) {
+  emitExperimentalWarning('globSync');
+  let patterns;
+  if (typeof pattern === 'object') {
+    validateStringArray(pattern, 'patterns');
+    patterns = pattern;
+  } else {
+    validateString(pattern, 'patterns');
+    patterns = [pattern];
+  }
+  const Glob = lazyGlob();
+  return new Glob(patterns, options).globSync();
+}
+
 module.exports = fs = {
   appendFile,
   appendFileSync,
@@ -3135,6 +3154,7 @@ module.exports = fs = {
   ftruncateSync,
   futimes,
   futimesSync,
+  globSync,
   lchown,
   lchownSync,
   lchmod: constants.O_SYMLINK !== undefined ? lchmod : undefined,
diff --git a/lib/internal/fs/glob.js b/lib/internal/fs/glob.js
index 4c62369048ce74..43f779027aade8 100644
--- a/lib/internal/fs/glob.js
+++ b/lib/internal/fs/glob.js
@@ -1,8 +1,10 @@
 'use strict';
 const { lstatSync, readdirSync } = require('fs');
+const { lstat, opendir } = require('fs/promises');
 const { join, resolve } = require('path');
 
 const {
+  createDeferredPromise,
   kEmptyObject,
 } = require('internal/util');
 const {
@@ -18,6 +20,7 @@ const {
   ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeSome,
+  PromisePrototypeThen,
   SafeMap,
   SafeSet,
   StringPrototypeEndsWith,
@@ -36,7 +39,17 @@ class Cache {
   #cache = new SafeMap();
   #statsCache = new SafeMap();
   #readdirCache = new SafeMap();
+  #opendirCache = new SafeMap();
 
+  stat(path) {
+    const cached = this.#statsCache.get(path);
+    if (cached) {
+      return cached;
+    }
+    const promise = PromisePrototypeThen(lstat(path), null, () => null);
+    this.#statsCache.set(path, promise);
+    return promise;
+  }
   statSync(path) {
     const cached = this.#statsCache.get(path);
     if (cached) {
@@ -54,6 +67,27 @@ class Cache {
   addToStatCache(path, val) {
     this.#statsCache.set(path, val);
   }
+  async* opendir(path) {
+    const cached = this.#opendirCache.get(path);
+    if (cached) {
+      yield* await cached;
+      return;
+    }
+    const { promise, resolve } = createDeferredPromise();
+    this.#opendirCache.set(path, promise);
+    let val;
+    try {
+      val = await opendir('./test');
+    } catch {
+      val = [];
+    }
+    const cache = [];
+    for await (const item of val) {
+      yield item;
+      cache.push(item);
+    }
+    resolve(cache);
+  }
   readdirSync(path) {
     const cached = this.#readdirCache.get(path);
     if (cached) {
@@ -151,6 +185,7 @@ class Glob {
   #results = new SafeSet();
   #queue = [];
   #subpatterns = new SafeMap();
+  #patterns;
   constructor(patterns, options = kEmptyObject) {
     validateObject(options, 'options');
     const { exclude, cwd } = options;
@@ -169,21 +204,18 @@ class Glob {
       platform: process.platform,
       nocaseMagicOnly: true,
     }));
+
+    this.#patterns = ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
+                                                                                         (pattern, i) => new Pattern(
+                                                                                           pattern,
+                                                                                           matcher.globParts[i],
+                                                                                           new SafeSet([0]),
+                                                                                           new SafeSet(),
+                                                                                         )));
   }
 
   globSync() {
-    ArrayPrototypePush(this.#queue, {
-      __proto__: null,
-      path: '.',
-      patterns: ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
-                                                                                    (pattern, i) => new Pattern(
-                                                                                      pattern,
-                                                                                      matcher.globParts[i],
-                                                                                      new SafeSet([0]),
-                                                                                      new SafeSet(),
-                                                                                    ))),
-    });
-
+    ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
     while (this.#queue.length > 0) {
       const item = ArrayPrototypePop(this.#queue);
       for (let i = 0; i < item.patterns.length; i++) {
@@ -376,6 +408,194 @@ class Glob {
       }
     }
   }
+
+
+  async* glob() {
+    ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
+    while (this.#queue.length > 0) {
+      const item = ArrayPrototypePop(this.#queue);
+      for (let i = 0; i < item.patterns.length; i++) {
+        yield* this.#iterateSubpatterns(item.path, item.patterns[i]);
+      }
+      this.#subpatterns
+        .forEach((patterns, path) => ArrayPrototypePush(this.#queue, { __proto__: null, path, patterns }));
+      this.#subpatterns.clear();
+    }
+    return ArrayFrom(this.#results);
+  }
+  async* #iterateSubpatterns(path, pattern) {
+    const seen = this.#cache.add(path, pattern);
+    if (seen) {
+      return;
+    }
+    const fullpath = resolve(this.#root, path);
+    const stat = await this.#cache.stat(fullpath);
+    const last = pattern.last;
+    const isDirectory = stat?.isDirectory() || (stat?.isSymbolicLink() && pattern.hasSeenSymlinks);
+    const isLast = pattern.isLast(isDirectory);
+    const isFirst = pattern.isFirst();
+
+    if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
+      // Absolute path, go to root
+      this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet([1])));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '') {
+      // Absolute path, go to root
+      this.#addSubpattern('/', pattern.child(new SafeSet([1])));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '..') {
+      // Start with .., go to parent
+      this.#addSubpattern('../', pattern.child(new SafeSet([1])));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '.') {
+      // Start with ., proceed
+      this.#addSubpattern('.', pattern.child(new SafeSet([1])));
+      return;
+    }
+
+    if (isLast && typeof pattern.at(-1) === 'string') {
+      // Add result if it exists
+      const p = pattern.at(-1);
+      const stat = await this.#cache.stat(join(fullpath, p));
+      if (stat && (p || isDirectory)) {
+        yield join(path, p);
+      }
+      if (pattern.indexes.size === 1 && pattern.indexes.has(last)) {
+        return;
+      }
+    } else if (isLast && pattern.at(-1) === lazyMinimatch().GLOBSTAR &&
+     (path !== '.' || pattern.at(0) === '.' || (last === 0 && stat))) {
+      // If pattern ends with **, add to results
+      // if path is ".", add it only if pattern starts with "." or pattern is exactly "**"
+      yield path;
+    }
+
+    if (!isDirectory) {
+      return;
+    }
+
+    let children;
+    const firstPattern = pattern.indexes.size === 1 && pattern.at(pattern.indexes.values().next().value);
+    if (typeof firstPattern === 'string') {
+      const stat = await this.#cache.stat(join(fullpath, firstPattern));
+      if (stat) {
+        stat.name = firstPattern;
+        children = [stat];
+      } else {
+        children = [];
+      }
+    } else {
+      children = this.#cache.opendir(fullpath);
+    }
+
+    for await (const entry of children) {
+      const entryPath = join(path, entry.name);
+      this.#cache.addToStatCache(join(fullpath, entry.name), entry);
+
+      const subPatterns = new SafeSet();
+      const nSymlinks = new SafeSet();
+      for (const index of pattern.indexes) {
+        // For each child, chek potential patterns
+        if (this.#cache.seen(entryPath, pattern, index) || this.#cache.seen(entryPath, pattern, index + 1)) {
+          return;
+        }
+        const current = pattern.at(index);
+        const nextIndex = index + 1;
+        const next = pattern.at(nextIndex);
+        const fromSymlink = pattern.symlinks.has(index);
+
+        if (current === lazyMinimatch().GLOBSTAR) {
+          if (entry.name[0] === '.' || (this.#exclude && this.#exclude(entry.name))) {
+            continue;
+          }
+          if (!fromSymlink && entry.isDirectory()) {
+            // If directory, add ** to its potential patterns
+            subPatterns.add(index);
+          } else if (!fromSymlink && index === last) {
+            // If ** is last, add to results
+            yield entryPath;
+          }
+
+          // Any pattern after ** is also a potential pattern
+          // so we can already test it here
+          const nextMatches = pattern.test(nextIndex, entry.name);
+          if (nextMatches && nextIndex === last && !isLast) {
+            // If next pattern is the last one, add to results
+            yield entryPath;
+          } else if (nextMatches && entry.isDirectory()) {
+            // Pattern mached, meaning two patterns forward
+            // are also potential patterns
+            // e.g **/b/c when entry is a/b - add c to potential patterns
+            subPatterns.add(index + 2);
+          }
+          if ((nextMatches || pattern.at(0) === '.') &&
+           (entry.isDirectory() || entry.isSymbolicLink()) && !fromSymlink) {
+            // If pattern after ** matches, or pattern starts with "."
+            // and entry is a directory or symlink, add to potential patterns
+            subPatterns.add(nextIndex);
+          }
+
+          if (entry.isSymbolicLink()) {
+            nSymlinks.add(index);
+          }
+
+          if (next === '..' && entry.isDirectory()) {
+            // In case pattern is "**/..",
+            // both parent and current directory should be added to the queue
+            // if this is the last pattern, add to results instead
+            const parent = join(path, '..');
+            if (nextIndex < last) {
+              if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) {
+                this.#subpatterns.set(path, [pattern.child(new SafeSet([nextIndex + 1]))]);
+              }
+              if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) {
+                this.#subpatterns.set(parent, [pattern.child(new SafeSet([nextIndex + 1]))]);
+              }
+            } else {
+              if (!this.#cache.seen(path, pattern, nextIndex)) {
+                this.#cache.add(path, pattern.child(new SafeSet([nextIndex])));
+                yield path;
+              }
+              if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
+                this.#cache.add(parent, pattern.child(new SafeSet([nextIndex])));
+                yield parent;
+              }
+            }
+          }
+        }
+        if (typeof current === 'string') {
+          if (pattern.test(index, entry.name) && index !== last) {
+            // If current pattern matches entry name
+            // the next pattern is a potential pattern
+            subPatterns.add(nextIndex);
+          } else if (current === '.' && pattern.test(nextIndex, entry.name)) {
+            // If current pattern is ".", proceed to test next pattern
+            if (nextIndex === last) {
+              yield entryPath;
+            } else {
+              subPatterns.add(nextIndex + 1);
+            }
+          }
+        }
+        if (typeof current === 'object' && pattern.test(index, entry.name)) {
+          // If current pattern is a regex that matches entry name (e.g *.js)
+          // add next pattern to potential patterns, or to results if it's the last pattern
+          if (index === last) {
+            yield entryPath;
+          } else if (entry.isDirectory()) {
+            subPatterns.add(nextIndex);
+          }
+        }
+      }
+      if (subPatterns.size > 0) {
+        // If there are potential patterns, add to queue
+        this.#addSubpattern(entryPath, pattern.child(subPatterns, nSymlinks));
+      }
+    }
+  }
 }
 
 module.exports = {
diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js
index 7f2582698dce21..e05dbe8ae73baf 100644
--- a/lib/internal/fs/promises.js
+++ b/lib/internal/fs/promises.js
@@ -84,16 +84,18 @@ const {
   validateInteger,
   validateObject,
   validateString,
+  validateStringArray,
   kValidateObjectAllowNullable,
 } = require('internal/validators');
 const pathModule = require('path');
 const { isAbsolute } = pathModule;
 const { toPathIfFileURL } = require('internal/url');
 const {
+  emitExperimentalWarning,
+  getLazy,
   kEmptyObject,
   lazyDOMException,
   promisify,
-  getLazy,
 } = require('internal/util');
 const EventEmitter = require('events');
 const { StringDecoder } = require('string_decoder');
@@ -1267,11 +1269,27 @@ async function* _watch(filename, options = kEmptyObject) {
   yield* watch(filename, options);
 }
 
+const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
+async function* glob(pattern, options) {
+  emitExperimentalWarning('glob');
+  let patterns;
+  if (typeof pattern === 'object') {
+    validateStringArray(pattern, 'patterns');
+    patterns = pattern;
+  } else {
+    validateString(pattern, 'patterns');
+    patterns = [pattern];
+  }
+  const Glob = lazyGlob();
+  yield* new Glob(patterns, options).globSync();
+}
+
 module.exports = {
   exports: {
     access,
     copyFile,
     cp,
+    glob,
     open,
     opendir: promisify(opendir),
     rename,
diff --git a/test/parallel/test-fs-glob.mjs b/test/parallel/test-fs-glob.mjs
index a789ed1167098a..98b7c23b51682d 100644
--- a/test/parallel/test-fs-glob.mjs
+++ b/test/parallel/test-fs-glob.mjs
@@ -1,11 +1,10 @@
-// Flags: --expose-internals
 import * as common from '../common/index.mjs';
 import tmpdir from '../common/tmpdir.js';
 import { resolve, dirname, sep } from 'node:path';
-import { mkdir, writeFile, symlink } from 'node:fs/promises';
-import { test } from 'node:test';
+import { mkdir, writeFile, symlink, glob } from 'node:fs/promises';
+import { globSync } from 'node:fs';
+import { test, describe } from 'node:test';
 import assert from 'node:assert';
-import glob from 'internal/fs/glob';
 
 tmpdir.refresh();
 
@@ -301,10 +300,24 @@ const patterns = {
   ],
 };
 
-for (const [pattern, expected] of Object.entries(patterns)) {
-  test(pattern, () => {
-    const actual = new glob.Glob([pattern], { cwd: fixtureDir }).globSync().sort();
-    const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
-    assert.deepStrictEqual(actual, normalized);
-  });
-}
+describe('globSync', function() {
+  for (const [pattern, expected] of Object.entries(patterns)) {
+    test(pattern, () => {
+      const actual = globSync(pattern, { cwd: fixtureDir }).sort();
+      const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
+      assert.deepStrictEqual(actual, normalized);
+    });
+  }
+});
+
+describe('glob', function() {
+  for (const [pattern, expected] of Object.entries(patterns)) {
+    test(pattern, async () => {
+      const actual = [];
+      for await (const item of glob(pattern, { cwd: fixtureDir })) actual.push(item);
+      actual.sort();
+      const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
+      assert.deepStrictEqual(actual, normalized);
+    });
+  }
+});
diff --git a/test/parallel/test-permission-fs-supported.js b/test/parallel/test-permission-fs-supported.js
index 1062117798b800..5797e191cd201a 100644
--- a/test/parallel/test-permission-fs-supported.js
+++ b/test/parallel/test-permission-fs-supported.js
@@ -90,6 +90,7 @@ const ignoreList = [
   ...syncAndAsyncAPI('readv'),
   ...syncAndAsyncAPI('write'),
   ...syncAndAsyncAPI('writev'),
+  ...syncAndAsyncAPI('glob'),
 ];
 
 {