diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
index 1d5fce3..d3bce6c 100644
--- a/.github/workflows/nodejs.yml
+++ b/.github/workflows/nodejs.yml
@@ -22,4 +22,9 @@ jobs:
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- - run: npm test
+ - name: check if json up to date
+ run: |
+ npm run xmltest.json
+ git add . # make sure line endings are sanitized by git
+ git diff HEAD --exit-code
+ - run: npm run test.zip && npm test
diff --git a/README.md b/README.md
index 007a253..5ccbe89 100644
--- a/README.md
+++ b/README.md
@@ -101,14 +101,23 @@ All methods have doc comments that include types.
- `getContent`
- `getEntries`
- `load`
+- `contentLoader`
+- `entriesLoader`
- `replaceWithWrappedCodePointAt`
- `replaceNonTextChars`
- `run`
(Feel free to contribute by automating the extraction of the documentation to this or another file.)
+### with different zip files
+
+The API can be used with other zip files by passing relative or absolute file names as arguments:
+- `load` (second argument)
+- `run` (first argument)
+
## Related Resources
- The page of the author linking to xmltest.zip:
-- THe way I found those testcases since they are part of a bigger testsuite for (Java SAX parsers)
+- The way I found those testcases since they are part of a bigger testsuite for (Java SAX parsers)
+- The W3C also provides an XML test suite: (the files in `xmltest.zip` are part of this but there is no clear license for the whole package)
- The PR that initially led to the creation of this package:
diff --git a/cache.js b/cache.js
new file mode 100644
index 0000000..e861074
--- /dev/null
+++ b/cache.js
@@ -0,0 +1,16 @@
+exports.cache = () => {
+ let map = new Map();
+
+ return {
+ clear: () => {
+ map = new Map();
+ },
+ delete: (key) => map.delete(key),
+ get: (key) => map.get(key),
+ has: (key) => map.has(key),
+ keys: () => [...map.keys()],
+ set: (key, value) => {
+ map.set(key, value);
+ },
+ };
+};
diff --git a/package.json b/package.json
index 48cbbd1..6a7204c 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,7 @@
"node": ">=10"
},
"files": [
+ "cache.js",
"README.md",
"LICENSE",
"xmltest.zip",
@@ -18,7 +19,9 @@
"scripts": {
"extract": "npx extract-zip xmltest.zip $PWD/data",
"test": "jest",
- "start": "jest --watch"
+ "test.zip": "./test.zip.sh",
+ "start": "npm run test -- --watch",
+ "xmltest.json": "runex ./xmltest.js > xmltest.json"
},
"repository": {
"type": "git",
diff --git a/test.zip b/test.zip
new file mode 100644
index 0000000..c32eb9c
Binary files /dev/null and b/test.zip differ
diff --git a/test.zip.sh b/test.zip.sh
new file mode 100755
index 0000000..6b6610b
--- /dev/null
+++ b/test.zip.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Creates test.zip file in the same directory
+
+# clear the folder if it already existed
+rm -rf data/folder
+# the `data` directory is .gitignored, so we can use it as temp
+mkdir -p data/folder
+# add one file that has content
+echo CONTENT > data/folder/file.ext
+# and one empty file
+touch data/folder/empty
+
+# It is important for the tests to have the optional `folder` entry in the zip file.
+(cd data && zip ../test.zip folder folder/*)
diff --git a/test/cache.test.js b/test/cache.test.js
new file mode 100644
index 0000000..b740279
--- /dev/null
+++ b/test/cache.test.js
@@ -0,0 +1,46 @@
+const { cache } = require("../cache");
+
+describe("cache", () => {
+ test("should have no keys initially", () => {
+ expect(cache().keys()).toHaveLength(0);
+ });
+ test.each(["key", 0, 1, "", null, NaN, undefined])(
+ "should store value for key `%s`",
+ (key) => {
+ const value = {};
+ const it = cache();
+
+ expect(it.has(key)).toBe(false);
+ it.set(key, value);
+ expect(it.has(key)).toBe(true);
+ expect(it.keys()).toHaveLength(1);
+ expect(it.get(key)).toBe(value);
+ }
+ );
+ test.each(["key", 0, 1, "", null, NaN, undefined])(
+ "should return undefined for key `%s`",
+ (key) => {
+ const it = cache();
+
+ expect(it.has(key)).toBe(false);
+ expect(it.get(key)).toBeUndefined();
+ }
+ );
+ test.each(["key", 0, 1, "", null, NaN, undefined])(
+ "should delete key for key `%s`",
+ (key) => {
+ const it = cache();
+ it.set(key, {});
+ it.delete(key);
+ expect(it.has(key)).toBe(false);
+ expect(it.get(key)).toBeUndefined();
+ }
+ );
+ test("should clear the cache", () => {
+ const it = cache();
+ it.set("key", {});
+ it.clear();
+ expect(it.has("key")).toBe(false);
+ expect(it.keys()).toHaveLength(0);
+ });
+});
diff --git a/test/run.test.js b/test/run.test.js
new file mode 100644
index 0000000..a019ff3
--- /dev/null
+++ b/test/run.test.js
@@ -0,0 +1,61 @@
+const entries = require("../xmltest.json");
+const { run, contentLoader, entriesLoader } = require("../xmltest.js");
+const path = require("path");
+
+const README_PATH = "xmltest/readme.html";
+const TEST_ZIP_PATH = path.join(__dirname, "..", "test.zip");
+
+const TEST_ZIP_ENTRIES = {
+ "folder/": "",
+ "folder/file.ext": "file.ext",
+ "folder/empty": "empty",
+};
+
+describe("run", () => {
+ beforeEach(contentLoader.CACHE.clear);
+ beforeEach(entriesLoader.CACHE.clear);
+ describe("only filter arguments", () => {
+ test("should return entries without any arguments", async () => {
+ // FYI: xmltest.zip doesn't contain any folder entries
+ expect(await run()).toEqual(entries);
+ expect(contentLoader.CACHE.keys()).toHaveLength(0);
+ expect(entriesLoader.CACHE.keys()).toHaveLength(1);
+ });
+ test("should return all (file) keys in entries with first argument 'xmltest'", async () => {
+ expect(Object.keys(await run("xmltest"))).toEqual(Object.keys(entries));
+ expect(contentLoader.CACHE.keys()).toHaveLength(1);
+ expect(entriesLoader.CACHE.keys()).toHaveLength(0);
+ });
+ test("should return the content of readme.html with first argument 'xmltest/readme.html'", async () => {
+ expect(await run(README_PATH)).toMatch(/^.*/);
+ });
+ test("should return dict with single key when multiple filters only match one entry", async () => {
+ const actual = await run(...README_PATH.split("/"));
+ expect(Object.keys(actual)).toHaveLength(1);
+ expect(actual[README_PATH]).toMatch(/^.*/);
+ });
+ });
+ describe("first argument is path to zip", () => {
+ test.each(["./test.zip", "../xmltest/test.zip", TEST_ZIP_PATH])(
+ "should return all entries without any filter arguments %s",
+ async (pathToZip) => {
+ expect(await run(pathToZip)).toEqual(TEST_ZIP_ENTRIES);
+ }
+ );
+ test("should return all file keys in entries with first filter argument 'folder'", async () => {
+ const actual = await run(TEST_ZIP_PATH, "folder");
+ expect(Object.keys(actual)).toEqual(
+ Object.keys(TEST_ZIP_ENTRIES).filter((entry) => !entry.endsWith("/"))
+ );
+ });
+ test("should return the content when first filter argument matches a file", async () => {
+ expect(await run(TEST_ZIP_PATH, "folder/file.ext")).toBe("CONTENT\n");
+ expect(await run(TEST_ZIP_PATH, "folder/empty")).toBe("");
+ });
+ test("should return dict with single key when multiple filters only match one entry", async () => {
+ const actual = await run(TEST_ZIP_PATH, "folder", "file");
+ expect(Object.keys(actual)).toHaveLength(1);
+ expect(actual["folder/file.ext"]).toMatch("CONTENT\n");
+ });
+ });
+});
diff --git a/xmltest.js b/xmltest.js
index c8cca42..1a123a4 100644
--- a/xmltest.js
+++ b/xmltest.js
@@ -1,13 +1,15 @@
-const entries = require('./xmltest.json')
const getStream = require('get-stream')
const path = require('path')
const {promisify} = require('util')
const yauzl = require('yauzl')
+
+const {cache} = require('./cache')
// for type definitions
const {Entry} = require('yauzl')
/**
- * @typedef PromiseResolve {function (response: typeof entries)}
+ * @typedef Entries {Record}
+ * @typedef PromiseResolve {function (response: Entries)}
* @typedef PromiseReject {function (reason: Error)}
* @typedef ReadFile {async function (response: Entry): Promise}
* @typedef EntryHandler {async function (response: Entry, readFile: ReadFile): Promise}
@@ -16,22 +18,18 @@ const {Entry} = require('yauzl')
*/
/**
- * Loads all file content from the zip file and caches it
+ * Loads all file content from the zip file.
*
* @param resolve {PromiseResolve}
* @param reject {PromiseReject}
* @returns {LoaderInstance}
*/
-const dataLoader = (resolve, reject) => {
- if (dataLoader.DATA) {
- resolve({...dataLoader.DATA})
- }
- /** @type {Partial} */
- const data = {}
+const contentLoader = (resolve, reject) => {
+ /** @type {Entries} */
+ const data = {};
const end = () => {
- dataLoader.DATA = data
- resolve(dataLoader.DATA)
+ resolve(data)
}
const entry = async (entry, readFile) => {
@@ -44,9 +42,9 @@ const dataLoader = (resolve, reject) => {
/**
* The module level cache for the zip file content.
*
- * @type {null | typeof entries}
+ * @type {null | Entries}
*/
-dataLoader.DATA = null
+contentLoader.CACHE = cache();
/**
* Loads the list of files and directories.
@@ -61,8 +59,8 @@ dataLoader.DATA = null
* @param reject {PromiseReject}
* @returns {LoaderInstance}
*/
-const jsonLoader = (resolve, reject) => {
- /** @type {Partial} */
+const entriesLoader = (resolve, reject) => {
+ /** @type {Entries} */
const data = {}
const end = () => {
resolve(data)
@@ -74,20 +72,28 @@ const jsonLoader = (resolve, reject) => {
}
return {end, entry}
}
+entriesLoader.CACHE = cache();
/**
- * Uses `loader` to iterate entries in a zipfile using `yauzl`.
+ * Uses `loader` to iterate entries in a zip file using `yauzl`.
+ * If `loader.CACHE` is set it is assumed to be an instance of `cache`,
+ * and is used to store the resolved result.
+ * If `loader.CACHE.has(location)` is true the zip file is not read again,
+ * since the cached result is returned.
+ * Use `loader.CACHE.delete(location)` or `loader.CACHE.clear()` when needed.
*
- * @see dataLoader
- * @see jsonLoader
+ * @see contentLoader
+ * @see contentLoader.CACHE
+ * @see entriesLoader
+ * @see entriesLoader.CACHE
*
- * @param loader {Loader} the loader to use (default: `dataLoader`)
+ * @param loader {Loader} the loader to use (default: `contentLoader`)
* @param location {string} absolute path to zip file (default: xmltest.zip)
- * @returns {Promise}
+ * @returns {Promise}
*/
-const load = async (loader = dataLoader, location = path.join(__dirname, 'xmltest.zip')) => {
- if (loader.DATA) {
- return {...loader.DATA}
+const load = async (loader = contentLoader, location = path.join(__dirname, 'xmltest.zip')) => {
+ if (loader.CACHE && loader.CACHE.has(location)) {
+ return {...loader.CACHE.get(location)}
}
const zipfile = await promisify(yauzl.open)(
@@ -95,13 +101,19 @@ const load = async (loader = dataLoader, location = path.join(__dirname, 'xmltes
)
const readFile = promisify(zipfile.openReadStream.bind(zipfile))
return new Promise((resolve, reject) => {
- const handler = loader(resolve, reject)
- zipfile.on('end', handler.end)
+ const resolver = loader.CACHE
+ ? (data) => {
+ loader.CACHE.set(location, data);
+ resolve(data);
+ }
+ : resolve;
+ const handler = loader(resolver, reject);
+ zipfile.on('end', handler.end);
zipfile.on('entry', async (entry) => {
- await handler.entry(entry, readFile)
- zipfile.readEntry()
- })
- zipfile.readEntry()
+ await handler.entry(entry, readFile);
+ zipfile.readEntry();
+ });
+ zipfile.readEntry();
})
}
@@ -216,9 +228,9 @@ const RELATED = {
* Filters `data` by applying `filters` to it's keys
*
* @see combineFilters
- * @param data {typeof entries}
+ * @param data {Entries}
* @param filters {(string | RegExp | Predicate)[]}
- * @returns {string | Partial} the value
+ * @returns {string | Entries} the value
* if the only filter only results a single entry,
* otherwise on object with all keys that match the filter.
*/
@@ -236,7 +248,7 @@ const getFiltered = (data, filters) => {
acc[key] = data[key]
return acc
},
- /** @type {Partial} */{}
+ /** @type {Entries} */{}
)
}
@@ -249,7 +261,7 @@ const getFiltered = (data, filters) => {
* @see load
*
* @param filters {string | RegExp | Predicate}
- * @returns {Promise>} the value
+ * @returns {Promise} the value
* if the only filter only results a single entry,
* otherwise on object with all keys that match the filter.
*/
@@ -260,30 +272,45 @@ const getContent = async (...filters) => getFiltered(await load(), filters)
*
* @see combineFilters
* @param filters {string | RegExp | Predicate}
- * @returns {string | Partial} the value
+ * @returns {string | Entries} the value
* if the only filter only results a single entry,
* otherwise on object with all keys that match the filter.
*/
-const getEntries = (...filters) => getFiltered(entries, filters)
+const getEntries = (...filters) => getFiltered(require('./xmltest.json')
+ , filters)
/**
* Makes module executable using `runex`.
- * With no arguments: Returns Object structure to store in `xmltest.json`
+ * If the first argument begins with `/`, `./` or `../` and ends with `.zip`,
+ * it is removed from the list of filter arguments and used as the path
+ * to the archive to load.
+ *
+ * With no filter arguments: Returns Object structure to store in `xmltest.json`
* `npx runex . > xmltest.json`
- * With one argument: Returns content string if exact key match,
+ * With one filter argument: Returns content string if exact key match,
* or content dict with filtered keys
- * With more arguments: Returns content dict with filtered keys
+ * With more filter arguments: Returns content dict with filtered keys
*
* @see getFiltered
* @see combineFilters
* @see load
+ * @see https://github.com/karfau/runex
*
- * @param filters {(string | RegExp | Predicate)[]}
- * @returns {Promise>}
+ * @param filters {string}
+ * @returns {Promise}
*/
-const run = async (...filters) => filters.length === 0
- ? getEntries()
- : getContent.apply(null, filters)
+const run = async (...filters) => {
+ let file;
+
+ if (filters.length > 0 && /^\.?\.?\/.*\.zip$/.test(filters[0])) {
+ file = filters.shift();
+ }
+
+ return getFiltered(
+ await load(filters.length === 0 ? entriesLoader : contentLoader, file),
+ filters
+ );
+};
const replaceWithWrappedCodePointAt = char => `{!${char.codePointAt(0).toString(16)}!}`
@@ -312,13 +339,15 @@ module.exports = {
getContent,
getEntries,
load,
+ contentLoader,
+ entriesLoader,
replaceNonTextChars,
replaceWithWrappedCodePointAt,
run
}
if (require.main === module) {
- // if you don't want to use `runex` just "launch" this module/package
- module.exports.run().then(console.log)
+ // if you don't want to use `runex` just "launch" this module/package:
+ // node xmltest ...
+ module.exports.run(...process.argv.slice(2)).then(console.log)
}
-