Skip to content

Commit

Permalink
refactor(cspell-io): URLs must be URLs and not filenames (#4984)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jason3S
Jason3S authored Nov 19, 2023
1 parent cf73832 commit 4e86c55
Show file tree
Hide file tree
Showing 18 changed files with 360 additions and 164 deletions.
1 change: 1 addition & 0 deletions cspell-dict.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ exonum
gimu
globby
globstar
gzipped
Harakat
jamstack
lcov
Expand Down
8 changes: 6 additions & 2 deletions packages/cspell-dictionary/src/SpellingDictionary/createSpellingDictionary.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { fileURLToPath } from 'node:url';

import { buildITrieFromWords, parseDictionaryLines } from 'cspell-trie-lib';
import { deepEqual } from 'fast-equals';

Expand Down Expand Up @@ -32,7 +34,7 @@ export function createSpellingDictionary(
source: string,
options?: SpellingDictionaryOptions | undefined,
): SpellingDictionary {
const params: CreateSpellingDictionaryParams = [wordList, name, source, options];
const params: CreateSpellingDictionaryParams = [wordList, name, source.toString(), options];

if (!Array.isArray(wordList)) {
return _createSpellingDictionary(params);
Expand Down Expand Up @@ -79,10 +81,12 @@ export interface SpellingDictionaryLoadError extends Error {

export function createFailedToLoadDictionary(
name: string,
source: string,
sourceUrl: URL | string,
error: Error,
options?: SpellingDictionaryOptions | undefined,
): SpellingDictionary {
const sourceHref = typeof sourceUrl === 'string' ? sourceUrl : sourceUrl.href;
const source = sourceHref.startsWith('file:') ? fileURLToPath(sourceUrl) : sourceHref;
options = options || {};
return {
name,
Expand Down
35 changes: 27 additions & 8 deletions packages/cspell-io/src/CSpellIO.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { BufferEncoding } from './models/BufferEncoding.js';
import type { FileReference, FileResource, UrlOrReference } from './models/FileResource.js';
import type { FileReference, FileResource, UrlOrFilename, UrlOrReference } from './models/FileResource.js';
import type { Stats } from './models/index.js';

export interface CSpellIO {
Expand Down Expand Up @@ -44,27 +44,46 @@ export interface CSpellIO {
* @returns 0 if they are equal and non-zero if they are not.
*/
compareStats(left: Stats, right: Stats): number;

/**
* Convert a string to a URL
* @param uriOrFilename - string or URL to convert.
* @param urlOrFilename - string or URL to convert.
* If it is a URL, then it is just returned as is.
* If is is a string and fully formed URL, then it is parsed.
* If it is a string without a protocol/scheme it is assumed to be relative to `relativeTo`.
* @param relativeTo - optional
*/
toURL(urlOrFilename: UrlOrReference, relativeTo?: string | URL): URL;

/**
* Convert a string to a File URL
* @param urlOrFilename - string or URL to convert.
* @param relativeTo - optional
*/
toURL(uriOrFilename: UrlOrReference): URL;
toFileURL(urlOrFilename: UrlOrFilename, relativeTo?: string | URL): URL;

/**
* Try to determine the base name of a URL.
*
* Note: this handles `data:` URLs with `filename` attribute:
*
* Example: `data:text/plain;charset=utf8;filename=hello.txt,Hello` would have a filename of `hello.txt`
* @param uriOrFilename - string or URL to extract the basename from.
* Example:
* - `data:text/plain;charset=utf8;filename=hello.txt,Hello` would have a filename of `hello.txt`
* - `file:///user/project/cspell.config.yaml` would have a filename of `cspell.config.yaml`
* - `https://raw.guc.com/sss/cspell/main/cspell.schema.json` would have a filename of `cspell.schema.json`
* @param urlOrFilename - string or URL to extract the basename from.
*/
uriBasename(uriOrFilename: UrlOrReference): string;
urlBasename(urlOrFilename: UrlOrReference): string;
/**
* Try to determine the directory URL of the uri.
*
* @param uriOrFilename - string or URL to extract the basename from.
* Example:
* - `file:///user/local/file.txt` becomes `file:///user/local/`
* - `file:///user/local/` becomes `file:///user/`
*
* @param urlOrFilename - string or URL
*/
uriDirname(uriOrFilename: UrlOrReference): URL;
urlDirname(urlOrFilename: UrlOrReference): URL;

// /**
// *
Expand Down
116 changes: 91 additions & 25 deletions packages/cspell-io/src/CSpellIONode.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { promises as fs } from 'fs';
import { basename } from 'path';
import { pathToFileURL } from 'url';
import { describe, expect, test } from 'vitest';

import { CSpellIONode } from './CSpellIONode.js';
import { toURL } from './node/file/url.js';
import { toFileURL } from './node/file/url.js';
import { makePathToFile, pathToSample as ps, pathToTemp } from './test/test.helper.js';

const sc = expect.stringContaining;
Expand All @@ -16,17 +17,36 @@ describe('CSpellIONode', () => {
});

test.each`
filename | baseFilename | content
${__filename} | ${basename(__filename)} | ${sc('This bit of text')}
${ps('cities.txt')} | ${'cities.txt'} | ${sc('San Francisco\n')}
${ps('cities.txt.gz')} | ${'cities.txt.gz'} | ${sc('San Francisco\n')}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt'} | ${'cities.txt'} | ${sc('San Francisco\n')}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'cities.txt.gz'} | ${sc('San Francisco\n')}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'hello.txt'} | ${sc('Hello, World!')}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${undefined} | ${sc('Hello, World!')}
filename | baseFilename | content
${__filename} | ${basename(__filename)} | ${sc('This bit of text')}
${ps('cities.txt')} | ${'cities.txt'} | ${sc('San Francisco\n')}
${ps('cities.txt.gz')} | ${'cities.txt.gz'} | ${sc('San Francisco\n')}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'hello.txt'} | ${sc('Hello, World!')}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${undefined} | ${sc('Hello, World!')}
`('readFile $filename', async ({ filename, content, baseFilename }) => {
const cspellIo = new CSpellIONode();
const url = toURL(filename);
const url = toFileURL(filename);
const expected = { url, content, baseFilename };
const result = await cspellIo.readFile(filename);
const gz = filename.endsWith('.gz') || undefined;
expect(result.url).toEqual(expected.url);
expect(result.getText()).toEqual(expected.content);
expect(result.baseFilename).toEqual(expected.baseFilename);
expect(!!result.gz).toEqual(!!gz);
});

const urlCities =
'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt';
const urlCitiesGz =
'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz';

test.runIf(process.env['TEST_FETCH']).each`
filename | baseFilename | content
${urlCities} | ${'cities.txt'} | ${sc('San Francisco\n')}
${urlCitiesGz} | ${'cities.txt.gz'} | ${sc('San Francisco\n')}
`('readFile https $filename', async ({ filename, content, baseFilename }) => {
const cspellIo = new CSpellIONode();
const url = toFileURL(filename);
const expected = { url, content, baseFilename };
const result = await cspellIo.readFile(filename);
const gz = filename.endsWith('.gz') || undefined;
Expand All @@ -43,7 +63,7 @@ describe('CSpellIONode', () => {
${'words.txt.gz'} | ${'one\ntwo\nthree\n'}
`('dataURL write/readFile $baseFilename', async ({ content, baseFilename }) => {
const cspellIo = new CSpellIONode();
const url = toURL(`data:text/plain,placeholder`);
const url = toFileURL(`data:text/plain,placeholder`);
const ref = await cspellIo.writeFile({ url, baseFilename }, content);
const result = await cspellIo.readFile(ref);
// console.error('dataURL %o', { ref, result });
Expand All @@ -53,16 +73,28 @@ describe('CSpellIONode', () => {
});

test.each`
filename | expected
${ps('cities.not_found.txt')} | ${oc({ code: 'ENOENT' })}
${ps('cities.not_found.txt.gz')} | ${oc({ code: 'ENOENT' })}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.not_found.txt'} | ${oc({ code: 'ENOENT' })}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/not_found/cities.txt.gz'} | ${oc({ code: 'ENOENT' })}
filename | expected
${ps('cities.not_found.txt')} | ${oc({ code: 'ENOENT' })}
${ps('cities.not_found.txt.gz')} | ${oc({ code: 'ENOENT' })}
`('readFile not found $filename', async ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
await expect(cspellIo.readFile(filename)).rejects.toEqual(expected);
});

const urlCitiesNotFound =
'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.not_found.txt';
const urlCitiesNotFoundGz =
'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/not_found/cities.txt.gz';

test.runIf(process.env['TEST_FETCH']).each`
filename | expected
${urlCitiesNotFound} | ${oc({ code: 'ENOENT' })}
${urlCitiesNotFoundGz} | ${oc({ code: 'ENOENT' })}
`('readFile not found https $filename', async ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
await expect(cspellIo.readFile(filename)).rejects.toEqual(expected);
});

test.each`
filename | content
${__filename} | ${sc('This bit of text')}
Expand All @@ -72,8 +104,8 @@ describe('CSpellIONode', () => {
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${sc('Hello, World!')}
`('readFileSync $filename', ({ filename, content }) => {
const cspellIo = new CSpellIONode();
const result = cspellIo.readFileSync({ url: toURL(filename) });
expect(result.url).toEqual(toURL(filename));
const result = cspellIo.readFileSync({ url: toFileURL(filename) });
expect(result.url).toEqual(toFileURL(filename));
expect(result.getText()).toEqual(content);
});

Expand Down Expand Up @@ -149,25 +181,59 @@ describe('CSpellIONode', () => {
});

test.each`
filename | expected
${ps('samples/cities.txt')} | ${sc('samples/cities.txt')}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'}
filename | expected
${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'}
`('toUrl $filename', ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.toURL(filename).toString()).toEqual(expected);
});

test.each`
filename | relativeTo | expected
${'samples/cities.txt'} | ${'file:///usr/local/'} | ${'file:///usr/local/samples/cities.txt'}
${'samples/cities.txt'} | ${'https://example.com'} | ${'https://example.com/samples/cities.txt'}
${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'file:///usr/local/'} | ${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'file:///usr/local/'} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
`('toUrl relativeTo $filename $relativeTo', ({ filename, relativeTo, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.toURL(filename, relativeTo).toString()).toEqual(expected);
});

test.each`
filename | expected
${ps('samples/cities.txt')} | ${sc('samples/cities.txt')}
${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'}
`('toFileUrl $filename', ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.toFileURL(filename).toString()).toEqual(expected);
});

test.each`
filename | relativeTo | expected
${ps('samples/cities.txt')} | ${'file:///usr/local/'} | ${sc('samples/cities.txt')}
${'samples/cities.txt'} | ${'file:///usr/local/'} | ${'file:///usr/local/samples/cities.txt'}
${'samples/cities.txt'} | ${ps()} | ${pathToFileURL(ps('samples/cities.txt')).toString()}
${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'file:///usr/local/'} | ${'https://raw.guc.com/sss/cspell/main/packages/cspell-io/samples/cities.txt.gz'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'file:///usr/local/'} | ${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'}
`('toFileUrl relativeTo $filename $relativeTo', ({ filename, relativeTo, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.toFileURL(filename, relativeTo).toString()).toEqual(expected);
});

test.each`
filename | expected
${ps('samples/cities.txt')} | ${'cities.txt'}
${'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-io/samples/cities.txt.gz'} | ${'cities.txt.gz'}
${'https://example.com/examples/code/'} | ${'code/'}
${'data:text/plain;charset=utf8;filename=hello.txt,Hello%2C%20World!'} | ${'hello.txt'}
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${'text.plain'}
`('uriBasename $filename', ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.uriBasename(filename)).toEqual(expected);
expect(cspellIo.urlBasename(toFileURL(filename))).toEqual(expected);
});

test.each`
Expand All @@ -178,6 +244,6 @@ describe('CSpellIONode', () => {
${'data:text/plain;charset=utf8;base64,SGVsbG8sIFdvcmxkISAlJSUlJCQkJCwsLCw'} | ${'data:'}
`('uriDirname $filename', ({ filename, expected }) => {
const cspellIo = new CSpellIONode();
expect(cspellIo.uriDirname(filename).toString()).toEqual(expected);
expect(cspellIo.urlDirname(toFileURL(filename)).toString()).toEqual(expected);
});
});
Loading

0 comments on commit 4e86c55

Please sign in to comment.