From 0aafeedf31fe5b991da989613205fc25b2116006 Mon Sep 17 00:00:00 2001 From: Andy Hayden Date: Wed, 13 Feb 2019 00:25:01 -0800 Subject: [PATCH 1/2] Add fs.walk and fs.walkSync --- fs/walk.ts | 134 ++++++++++++++++++++++++ fs/walk_test.ts | 265 ++++++++++++++++++++++++++++++++++++++++++++++++ test.ts | 1 + 3 files changed, 400 insertions(+) create mode 100644 fs/walk.ts create mode 100644 fs/walk_test.ts diff --git a/fs/walk.ts b/fs/walk.ts new file mode 100644 index 000000000000..46b631a175ef --- /dev/null +++ b/fs/walk.ts @@ -0,0 +1,134 @@ +import { + FileInfo, + cwd, + readDir, + readDirSync, + readlink, + readlinkSync, + stat, + statSync +} from "deno"; +import { relative } from "path.ts"; + +export interface WalkOptions { + maxDepth?: number; + exts?: string[]; + match?: RegExp[]; + skip?: RegExp[]; + // FIXME don't use `any` here? + onError?: (err: any) => void; + followSymlinks?: Boolean; +} + +/** Generate all files in a directory recursively. + * + * for async const fileInfo of walk() { + * console.log(fileInfo.path); + * assert(fileInfo.isFile()); + * }; + */ +export async function* walk( + dir: string = ".", + options: WalkOptions = {} +): AsyncIterableIterator { + options.maxDepth -= 1; + let ls: FileInfo[] = []; + try { + ls = await readDir(dir); + } catch (err) { + if (options.onError) { + options.onError(err); + } + } + for (let f of ls) { + if (f.isSymlink()) { + if (options.followSymlinks) { + f = await resolve(f); + } else { + continue; + } + } + if (f.isFile()) { + if (include(f, options)) { + yield f; + } + } else { + if (!(options.maxDepth < 0)) { + yield* walk(f.path, options); + } + } + } +} + +/** Generate all files in a directory recursively. + * + * for const fileInfo of walkSync() { + * console.log(fileInfo.path); + * assert(fileInfo.isFile()); + * }; + */ +export function* walkSync( + dir: string = ".", + options: WalkOptions = {} +): IterableIterator { + options.maxDepth -= 1; + let ls: FileInfo[] = []; + try { + ls = readDirSync(dir); + } catch (err) { + if (options.onError) { + options.onError(err); + } + } + for (let f of ls) { + if (f.isSymlink()) { + if (options.followSymlinks) { + f = resolveSync(f); + } else { + continue; + } + } + if (f.isFile()) { + if (include(f, options)) { + yield f; + } + } else { + if (!(options.maxDepth < 0)) { + yield* walkSync(f.path, options); + } + } + } +} + +function include(f: FileInfo, options: WalkOptions): Boolean { + if (options.exts && !options.exts.some(ext => f.path.endsWith(ext))) { + return false; + } + if (options.match && !options.match.some(pattern => pattern.test(f.path))) { + return false; + } + if (options.skip && options.skip.some(pattern => pattern.test(f.path))) { + return false; + } + return true; +} + +async function resolve(f: FileInfo): Promise { + // This is the full path, unfortunately if we were to make it relative + // it could resolve to a symlink and cause an infinite loop. + const fpath = await readlink(f.path); + f = await stat(fpath); + // workaround path not being returned by stat + f.path = fpath; + return f; +} + +function resolveSync(f: FileInfo): FileInfo { + // This is the full path, unfortunately if we were to make it relative + // it could resolve to a symlink and cause an infinite loop. + const fpath = readlinkSync(f.path); + f = statSync(fpath); + // workaround path not being returned by stat + f.path = fpath; + return f; +} diff --git a/fs/walk_test.ts b/fs/walk_test.ts new file mode 100644 index 000000000000..7f21a2fb40d5 --- /dev/null +++ b/fs/walk_test.ts @@ -0,0 +1,265 @@ +import { + cwd, + chdir, + FileInfo, + makeTempDir, + mkdir, + open, + platform, + remove, + symlink +} from "deno"; + +import { walk, walkSync, WalkOptions } from "./walk.ts"; +import { test, assert, TestFunction } from "../testing/mod.ts"; + +const isWindows = platform.os === "win"; + +async function testWalk( + setup: (string) => void | Promise, + t: TestFunction +): Promise { + const name = t.name; + async function fn() { + const orig_cwd = cwd(); + const d = await makeTempDir(); + chdir(d); + try { + await setup(d); + await t(); + } finally { + chdir(orig_cwd); + remove(d, { recursive: true }); + } + } + test({ name, fn }); +} + +async function walkArray( + dirname: string = ".", + options: WalkOptions = {} +): Promise> { + const arr: string[] = []; + for await (const f of walk(dirname, { ...options })) { + arr.push(f.path.replace(/\\/g, "/")); + } + arr.sort(); + const arr_sync = Array.from(walkSync(dirname, options), (f: FileInfo) => + f.path.replace(/\\/g, "/") + ).sort(); + assert.equal(arr, arr_sync); + return arr; +} + +async function touch(path: string): Promise { + await open(path, "w"); +} +function assertReady(expectedLength: number) { + const arr = Array.from(walkSync(), (f: FileInfo) => f.path); + assert.equal(arr.length, expectedLength); +} + +testWalk( + async (d: string) => { + await mkdir(d + "/empty"); + }, + async function emptyDir() { + const arr = await walkArray(); + assert.equal(arr.length, 0); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + }, + async function singleFile() { + const arr = await walkArray(); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./x"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + }, + async function iteratable() { + let count = 0; + for (const f of walkSync()) { + count += 1; + } + assert.equal(count, 1); + for await (const f of walk()) { + count += 1; + } + assert.equal(count, 2); + } +); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await touch(d + "/a/x"); + }, + async function nestedSingleFile() { + const arr = await walkArray(); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./a/x"); + } +); + +testWalk( + async (d: string) => { + await mkdir(d + "/a/b/c/d", true); + await touch(d + "/a/b/c/d/x"); + }, + async function depth() { + assertReady(1); + const arr_3 = await walkArray(".", { maxDepth: 3 }); + assert.equal(arr_3.length, 0); + const arr_5 = await walkArray(".", { maxDepth: 5 }); + assert.equal(arr_5.length, 1); + assert.equal(arr_5[0], "./a/b/c/d/x"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x.ts"); + await touch(d + "/y.rs"); + }, + async function ext() { + assertReady(2); + const arr = await walkArray(".", { exts: [".ts"] }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./x.ts"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x.ts"); + await touch(d + "/y.rs"); + await touch(d + "/z.py"); + }, + async function extAny() { + assertReady(3); + const arr = await walkArray(".", { exts: [".rs", ".ts"] }); + assert.equal(arr.length, 2); + assert.equal(arr[0], "./x.ts"); + assert.equal(arr[1], "./y.rs"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + await touch(d + "/y"); + }, + async function match() { + assertReady(2); + const arr = await walkArray(".", { match: [/x/] }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./x"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + await touch(d + "/y"); + await touch(d + "/z"); + }, + async function matchAny() { + assertReady(3); + const arr = await walkArray(".", { match: [/x/, /y/] }); + assert.equal(arr.length, 2); + assert.equal(arr[0], "./x"); + assert.equal(arr[1], "./y"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + await touch(d + "/y"); + }, + async function skip() { + assertReady(2); + const arr = await walkArray(".", { skip: [/x/] }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./y"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x"); + await touch(d + "/y"); + await touch(d + "/z"); + }, + async function skipAny() { + assertReady(3); + const arr = await walkArray(".", { skip: [/x/, /y/] }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./z"); + } +); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await mkdir(d + "/b"); + await touch(d + "/a/x"); + await touch(d + "/a/y"); + await touch(d + "/b/z"); + }, + async function subDir() { + assertReady(3); + const arr = await walkArray("b"); + assert.equal(arr.length, 1); + assert.equal(arr[0], "b/z"); + } +); + +testWalk(async (d: string) => {}, async function onError() { + assertReady(0); + const ignored = await walkArray("missing"); + assert.equal(ignored.length, 0); + let errors = 0; + const arr = await walkArray("missing", { onError: e => (errors += 1) }); + // It's 2 since walkArray iterates over both sync and async. + assert.equal(errors, 2); +}); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await mkdir(d + "/b"); + await touch(d + "/a/x"); + await touch(d + "/a/y"); + await touch(d + "/b/z"); + try { + await symlink(d + "/b", d + "/a/bb"); + } catch (err) { + assert(isWindows); + assert(err.message, "Not implemented"); + } + }, + async function symlink() { + // symlink is not yet implemented on Windows. + if (isWindows) { + return; + } + + assertReady(3); + const files = await walkArray("a"); + assert.equal(files.length, 2); + assert(!files.includes("a/bb/z")); + + const arr = await walkArray("a", { followSymlinks: true }); + assert.equal(arr.length, 3); + assert(arr.some(f => f.endsWith("/b/z"))); + } +); diff --git a/test.ts b/test.ts index da3cecf3c08c..e94bdcee67f1 100755 --- a/test.ts +++ b/test.ts @@ -11,6 +11,7 @@ import "io/util_test.ts"; import "io/writers_test.ts"; import "io/readers_test.ts"; import "fs/path/test.ts"; +import "fs/walk_test.ts"; import "io/test.ts"; import "http/server_test.ts"; import "http/file_server_test.ts"; From 938485277e40e43787ad4eed326b88aeb6f97271 Mon Sep 17 00:00:00 2001 From: Andy Hayden Date: Wed, 13 Feb 2019 17:51:55 -0800 Subject: [PATCH 2/2] Fix code in jsdoc --- fs/walk.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/walk.ts b/fs/walk.ts index 46b631a175ef..92e4ba593f07 100644 --- a/fs/walk.ts +++ b/fs/walk.ts @@ -22,7 +22,7 @@ export interface WalkOptions { /** Generate all files in a directory recursively. * - * for async const fileInfo of walk() { + * for await (const fileInfo of walk()) { * console.log(fileInfo.path); * assert(fileInfo.isFile()); * }; @@ -62,7 +62,7 @@ export async function* walk( /** Generate all files in a directory recursively. * - * for const fileInfo of walkSync() { + * for (const fileInfo of walkSync()) { * console.log(fileInfo.path); * assert(fileInfo.isFile()); * };