From fd7a896be53dc5353ad719c093e15157391e2bb1 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Thu, 29 Dec 2022 13:29:22 -0800 Subject: [PATCH 1/2] fix: allow traversal of symlinks in glob Closes #2130 This adds a `followSymlinks` option to the `glob()` function, and enables it when searching for entry points. If `true`, and a symbolic link is encountered, a stat will be taken of the symlink's target. If a dir or file, the symlink is handled like any other dir or file. If a symbolic link itself (symlink to a symlink), we take a stat of the next until we find a dir or file, then handle it. There is a little bit of caching to avoid extra I/O, and protection from recursive symlinks. However, it's possible (though unlikely) that the FS could cause a "max call stack exceeded" exception. If someone runs into this, we can change the implementation to a loop instead of a recursive function. Apologies for blasting the `do..while`. I love a good `do` myself, but splitting out the lambda functions make it untenable. --- src/lib/utils/entry-point.ts | 2 +- src/lib/utils/fs.ts | 93 ++++++++++++++++++++++++++++-------- src/test/utils/fs.test.ts | 88 +++++++++++++++++++++++++++++++++- 3 files changed, 159 insertions(+), 24 deletions(-) diff --git a/src/lib/utils/entry-point.ts b/src/lib/utils/entry-point.ts index 55c33d9b6..8cc6fd5ec 100644 --- a/src/lib/utils/entry-point.ts +++ b/src/lib/utils/entry-point.ts @@ -203,7 +203,7 @@ export function getExpandedEntryPointsForPaths( function expandGlobs(inputFiles: string[]) { const base = getCommonDirectory(inputFiles); const result = inputFiles.flatMap((entry) => - glob(entry, base, { includeDirectories: true }) + glob(entry, base, { includeDirectories: true, followSymlinks: true }) ); return result; } diff --git a/src/lib/utils/fs.ts b/src/lib/utils/fs.ts index 256e2419f..c5ded7fe7 100644 --- a/src/lib/utils/fs.ts +++ b/src/lib/utils/fs.ts @@ -1,7 +1,7 @@ import * as fs from "fs"; import { promises as fsp } from "fs"; import { Minimatch } from "minimatch"; -import { dirname, join } from "path"; +import { dirname, join, relative } from "path"; /** * Get the longest directory path common to all files. @@ -139,16 +139,75 @@ export function copySync(src: string, dest: string): void { export function glob( pattern: string, root: string, - options?: { includeDirectories?: boolean } + options: { includeDirectories?: boolean; followSymlinks?: boolean } = {} ): string[] { const result: string[] = []; const mini = new Minimatch(normalizePath(pattern)); const dirs: string[][] = [normalizePath(root).split("/")]; + // cache of real paths to avoid infinite recursion + const symlinkTargetsSeen: Set = new Set(); + // cache of fs.realpathSync results to avoid extra I/O + const realpathCache: Map = new Map(); + const { includeDirectories = false, followSymlinks = false } = options; + + let dir = dirs.shift(); + + const handleFile = (path: string) => { + const childPath = [...dir!, path].join("/"); + if (mini.match(childPath)) { + result.push(childPath); + } + }; + + const handleDirectory = (path: string) => { + const childPath = [...dir!, path]; + if ( + mini.set.some((row) => + mini.matchOne(childPath, row, /* partial */ true) + ) + ) { + dirs.push(childPath); + } + }; + + const handleSymlink = (path: string) => { + const childPath = [...dir!, path].join("/"); + let realpath: string; + try { + realpath = + realpathCache.get(childPath) ?? fs.realpathSync(childPath); + realpathCache.set(childPath, realpath); + } catch { + return; + } - do { - const dir = dirs.shift()!; + if (symlinkTargetsSeen.has(realpath)) { + return; + } + symlinkTargetsSeen.add(realpath); + + try { + const stats = fs.statSync(realpath); + if (stats.isDirectory()) { + handleDirectory(path); + } else if (stats.isFile()) { + handleFile(path); + } else if (stats.isSymbolicLink()) { + const dirpath = dir!.join("/"); + if (dirpath === realpath) { + // special case: real path of symlink is the directory we're currently traversing + return; + } + const targetPath = relative(dirpath, realpath); + handleSymlink(targetPath); + } // everything else should be ignored + } catch (e) { + // invalid symbolic link; ignore + } + }; - if (options?.includeDirectories && mini.match(dir.join("/"))) { + while (dir) { + if (includeDirectories && mini.match(dir.join("/"))) { result.push(dir.join("/")); } @@ -156,24 +215,16 @@ export function glob( withFileTypes: true, })) { if (child.isFile()) { - const childPath = [...dir, child.name].join("/"); - if (mini.match(childPath)) { - result.push(childPath); - } - } - - if (child.isDirectory() && child.name !== "node_modules") { - const childPath = dir.concat(child.name); - if ( - mini.set.some((row) => - mini.matchOne(childPath, row, /* partial */ true) - ) - ) { - dirs.push(childPath); - } + handleFile(child.name); + } else if (child.isDirectory() && child.name !== "node_modules") { + handleDirectory(child.name); + } else if (followSymlinks && child.isSymbolicLink()) { + handleSymlink(child.name); } } - } while (dirs.length); + + dir = dirs.shift(); + } return result; } diff --git a/src/test/utils/fs.test.ts b/src/test/utils/fs.test.ts index 0b8bca05e..756a69beb 100644 --- a/src/test/utils/fs.test.ts +++ b/src/test/utils/fs.test.ts @@ -1,6 +1,8 @@ +import * as fs from "fs"; +import { createServer } from "net"; import { Project, tempdirProject } from "@typestrong/fs-fixture-builder"; -import { deepStrictEqual as equal } from "assert"; -import { basename } from "path"; +import { AssertionError, deepStrictEqual as equal } from "assert"; +import { basename, dirname, resolve } from "path"; import { glob } from "../../lib/utils/fs"; describe("fs.ts", () => { @@ -37,5 +39,87 @@ describe("fs.ts", () => { ["test.ts", "test2.ts"] ); }); + + describe("when 'followSymlinks' option is true", () => { + it("should navigate symlinked directories", () => { + const target = dirname(fix.dir("a").addFile("test.ts").path); + fix.write(); + fs.symlinkSync(target, resolve(fix.cwd, "b"), "junction"); + equal( + glob(`${fix.cwd}/b/*.ts`, fix.cwd, { + followSymlinks: true, + }).map((f) => basename(f)), + ["test.ts"] + ); + }); + + it("should navigate recursive symlinked directories only once", () => { + fix.addFile("test.ts"); + fix.write(); + fs.symlinkSync( + fix.cwd, + resolve(fix.cwd, "recursive"), + "junction" + ); + equal( + glob(`${fix.cwd}/**/*.ts`, fix.cwd, { + followSymlinks: true, + }).map((f) => basename(f)), + ["test.ts", "test.ts"] + ); + }); + + it("should handle symlinked files", function () { + const { path } = fix.addFile("test.ts"); + fix.write(); + try { + fs.symlinkSync( + path, + resolve(dirname(path), "test-2.ts"), + "file" + ); + } catch (err) { + // on windows, you need elevated permissions to create a file symlink. + // maybe we have them! maybe we don't! + if ( + (err as NodeJS.ErrnoException).code === "EPERM" && + process.platform === "win32" + ) { + return this.skip(); + } + } + equal( + glob(`${fix.cwd}/**/*.ts`, fix.cwd, { + followSymlinks: true, + }).map((f) => basename(f)), + ["test-2.ts", "test.ts"] + ); + }); + }); + + it("should ignore anything that is not a file, symbolic link, or directory", function (done) { + // Use unix socket for example, because that's easiest to create. + // Skip on Windows because it doesn't support unix sockets + if (process.platform === "win32") { + return this.skip(); + } + fix.write(); + + const sockServer = createServer() + .unref() + .listen(resolve(fix.cwd, "socket.sock")) + .once("listening", () => { + let err: AssertionError | null = null; + try { + equal(glob(`${fix.cwd}/*.sock`, fix.cwd), []); + } catch (e) { + err = e as AssertionError; + } finally { + sockServer.close(() => { + done(err); + }); + } + }); + }); }); }); From 6c1b4ac811c2bb2c5b91fe43b61cc06b5bd90522 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Wed, 4 Jan 2023 10:36:12 -0800 Subject: [PATCH 2/2] fix(test): glob root match test matches against platform-normalized path --- src/test/utils/fs.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/utils/fs.test.ts b/src/test/utils/fs.test.ts index 756a69beb..7dee6ef87 100644 --- a/src/test/utils/fs.test.ts +++ b/src/test/utils/fs.test.ts @@ -2,7 +2,7 @@ import * as fs from "fs"; import { createServer } from "net"; import { Project, tempdirProject } from "@typestrong/fs-fixture-builder"; import { AssertionError, deepStrictEqual as equal } from "assert"; -import { basename, dirname, resolve } from "path"; +import { basename, dirname, resolve, normalize } from "path"; import { glob } from "../../lib/utils/fs"; describe("fs.ts", () => { @@ -20,7 +20,7 @@ describe("fs.ts", () => { fix.write(); const result = glob(fix.cwd, fix.cwd, { includeDirectories: true }); - equal(result, [fix.cwd]); + equal(result.map(normalize), [fix.cwd].map(normalize)); }); it("Handles basic globbing", () => {