diff --git a/packages/cspell-tools/cspell-tools.config.schema.json b/packages/cspell-tools/cspell-tools.config.schema.json index a801951ad0f1..cab706be74a8 100644 --- a/packages/cspell-tools/cspell-tools.config.schema.json +++ b/packages/cspell-tools/cspell-tools.config.schema.json @@ -209,6 +209,13 @@ ], "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary." }, + "checksumFile": { + "description": "Path to checksum file. `true` - defaults to `./checksum.txt`.", + "type": [ + "string", + "boolean" + ] + }, "generateNonStrict": { "default": true, "description": "Generate lower case / accent free versions of words.", diff --git a/packages/cspell-tools/package.json b/packages/cspell-tools/package.json index 294082943dac..1c4f88a9b0b6 100644 --- a/packages/cspell-tools/package.json +++ b/packages/cspell-tools/package.json @@ -16,12 +16,11 @@ "watch": "tsc -p . -w", "clean-build": "pnpm run clean && pnpm run build", "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"", - "coverage": "pnpm coverage:vitest && pnpm coverage:fix", + "coverage": "pnpm coverage:vitest", "coverage:vitest": "vitest run --coverage", - "coverage:fix": "nyc report --temp-dir \"$(pwd)/coverage\" --reporter lcov --report-dir \"$(pwd)/coverage\" --cwd ../..", - "test-watch": "jest --watch", + "test:watch": "vitest", "test": "vitest run", - "update-snapshot": "jest --updateSnapshot" + "update-snapshot": "vitest run -u" }, "repository": { "type": "git", diff --git a/packages/cspell-tools/src/app.ts b/packages/cspell-tools/src/app.ts index b288115e4b18..b2212ad34626 100644 --- a/packages/cspell-tools/src/app.ts +++ b/packages/cspell-tools/src/app.ts @@ -110,6 +110,7 @@ export async function run(program: program.Command, argv: string[], flags?: Feat .command('build [targets...]') .description('Build the targets defined in the run configuration.') .option('-c, --config ', 'Specify the run configuration file.') + .option('--conditional', 'Conditional build.') .option('-r, --root ', 'Specify the run directory') .action(build); diff --git a/packages/cspell-tools/src/build.ts b/packages/cspell-tools/src/build.ts index eaf20086a7a5..9527e423b418 100644 --- a/packages/cspell-tools/src/build.ts +++ b/packages/cspell-tools/src/build.ts @@ -14,6 +14,9 @@ export interface BuildOptions { /** Current working directory */ cwd?: string | undefined; + + /** Conditional build based upon the targets matching the `checksum.txt` file. */ + conditional?: boolean; } const moduleName = 'cspell-tools'; @@ -46,11 +49,15 @@ export async function build(targets: string[] | undefined, options: BuildOptions throw 'cspell-tools.config not found.'; } - const buildInfo: CompileRequest = normalizeRequest(config.config, options.root || path.dirname(config.filepath)); - await compile(buildInfo, { filter, cwd: options.cwd }); + const configDir = path.dirname(config.filepath); + const buildInfo: CompileRequest = normalizeRequest( + config.config, + path.resolve(configDir, options.root || configDir), + ); + await compile(buildInfo, { filter, cwd: options.cwd, conditionalBuild: options.conditional || false }); } function normalizeRequest(buildInfo: CompileRequest, root: string): CompileRequest { - const { rootDir = root, targets = [] } = buildInfo; - return { rootDir, targets }; + const { rootDir = root, targets = [], checksumFile } = buildInfo; + return { rootDir: path.resolve(rootDir), targets, checksumFile }; } diff --git a/packages/cspell-tools/src/compiler/__snapshots__/compile.test.ts.snap b/packages/cspell-tools/src/compiler/__snapshots__/compile.test.ts.snap index 84f540c9f53f..6c2b8707df76 100644 --- a/packages/cspell-tools/src/compiler/__snapshots__/compile.test.ts.snap +++ b/packages/cspell-tools/src/compiler/__snapshots__/compile.test.ts.snap @@ -152,3 +152,156 @@ Error+ msg " `; + +exports[`compile > compile conditional 'cities.txt' fmt: 'plaintext' gz: false alt: true 1`] = ` +" +# cspell-tools: keep-case no-split + +London +Los Angeles +Mexico City +New Amsterdam +New Delhi +New York +Paris +San Francisco +~london +~los angeles +~mexico city +~new amsterdam +~new delhi +~new york +~paris +~san francisco +" +`; + +exports[`compile > compile conditional 'cities.txt' fmt: 'plaintext' gz: false alt: undefined 1`] = ` +" +# cspell-tools: keep-case no-split + +London +Los Angeles +Mexico City +New Amsterdam +New Delhi +New York +Paris +San Francisco +" +`; + +exports[`compile > compile conditional 'cities.txt' fmt: 'plaintext' gz: true alt: true 1`] = ` +" +# cspell-tools: keep-case no-split + +London +Los Angeles +Mexico City +New Amsterdam +New Delhi +New York +Paris +San Francisco +~london +~los angeles +~mexico city +~new amsterdam +~new delhi +~new york +~paris +~san francisco +" +`; + +exports[`compile > compile conditional 'cities.txt' fmt: 'plaintext' gz: true alt: undefined 1`] = ` +" +# cspell-tools: keep-case no-split + +London +Los Angeles +Mexico City +New Amsterdam +New Delhi +New York +Paris +San Francisco +" +`; + +exports[`compile > compile conditional 'cities.txt' fmt: 'trie3' gz: false alt: undefined 1`] = ` +"#!/usr/bin/env cspell-trie reader +TrieXv3 +base=10 +# Built by cspell-tools. +# Data: +__DATA__ +L +o +ndon$4s Angeles$9<2 +M +e +xico City$9<2 +N +e +w Amsterdam$9Delhi$5York$8 +P +a +ri#13;<4 +S +a +n Francisco$9<4 +" +`; + +exports[`compile > compile conditional 'sampleCodeDic.txt' fmt: 'plaintext' gz: false alt: true 1`] = ` +" +# cspell-tools: keep-case no-split + +!Codemsg +!Errorerror +!codecode +!err ++code ++code+ ++error ++error+ ++msg +Café +Code +Code+ +Error +Error+ +msg +~!codemsg +~!errorerror +~cafe +~café +~code +~code+ +~error +~error+ +" +`; + +exports[`compile > compile conditional 'sampleCodeDic.txt' fmt: 'plaintext' gz: false alt: undefined 1`] = ` +" +# cspell-tools: keep-case no-split + +!Codemsg +!Errorerror +!codecode +!err ++code ++code+ ++error ++error+ ++msg +Café +Code +Code+ +Error +Error+ +msg +" +`; diff --git a/packages/cspell-tools/src/compiler/compile.test.ts b/packages/cspell-tools/src/compiler/compile.test.ts index 056e840c5135..204f3ccbf4a4 100644 --- a/packages/cspell-tools/src/compiler/compile.test.ts +++ b/packages/cspell-tools/src/compiler/compile.test.ts @@ -6,6 +6,7 @@ import { spyOnConsole } from '../test/console.js'; import { createTestHelper } from '../test/TestHelper.js'; import { compile } from './compile.js'; import { readTextFile } from './readers/readTextFile.js'; +import { checkShasumFile } from '../shasum/shasum.js'; const testHelper = createTestHelper(import.meta.url); @@ -60,6 +61,49 @@ describe('compile', () => { expect(content).toMatchSnapshot(); }, ); + + test.each` + file | format | compress | generateNonStrict + ${'cities.txt'} | ${'plaintext'} | ${false} | ${true} + ${'cities.txt'} | ${'plaintext'} | ${true} | ${true} + ${'cities.txt'} | ${'plaintext'} | ${false} | ${undefined} + ${'cities.txt'} | ${'plaintext'} | ${true} | ${undefined} + ${'cities.txt'} | ${'trie3'} | ${false} | ${undefined} + ${'sampleCodeDic.txt'} | ${'plaintext'} | ${false} | ${undefined} + ${'sampleCodeDic.txt'} | ${'plaintext'} | ${false} | ${true} + `( + 'compile conditional $file fmt: $format gz: $compress alt: $generateNonStrict', + async ({ format, file, generateNonStrict, compress }) => { + const targetDirectory = t(`.`); + const target: Target = { + name: 'myDictionary', + targetDirectory, + format, + sources: [sample(file)], + compress, + generateNonStrict, + trieBase: 10, + sort: true, + }; + const req: CompileRequest = { + targets: [target], + rootDir: targetDirectory, + checksumFile: true, + }; + + await compile(req, { conditionalBuild: true }); + + const ext = (format === 'plaintext' ? '.txt' : '.trie') + ((compress && '.gz') || ''); + const content = await readTextFile(`${targetDirectory}/myDictionary${ext}`); + expect(content).toMatchSnapshot(); + const check = await checkShasumFile(path.join(targetDirectory, 'checksum.txt'), [], targetDirectory); + expect(check.passed).toBe(true); + + await compile(req, { conditionalBuild: true }); + const check2 = await checkShasumFile(path.join(targetDirectory, 'checksum.txt'), [], targetDirectory); + expect(check2.passed).toBe(true); + }, + ); }); function t(...parts: string[]): string { diff --git a/packages/cspell-tools/src/compiler/compile.ts b/packages/cspell-tools/src/compiler/compile.ts index ddb6e38be03f..57e048e5cdaf 100644 --- a/packages/cspell-tools/src/compiler/compile.ts +++ b/packages/cspell-tools/src/compiler/compile.ts @@ -5,14 +5,15 @@ import * as path from 'path'; import type { CompileRequest, - CompileSourceOptions, - CompileTargetOptions, + CompileSourceOptions as CompileSourceConfig, + CompileTargetOptions as CompileTargetConfig, DictionarySource, FilePath, FileSource, Target, } from '../config/index.js'; import { isFileListSource, isFilePath, isFileSource } from '../config/index.js'; +import { checkShasumFile, updateChecksumForFiles } from '../shasum/index.js'; import { createAllowedSplitWordsFromFiles } from './createWordsCollection.js'; import { logWithTimestamp } from './logWithTimestamp.js'; import { readTextFile } from './readers/readTextFile.js'; @@ -31,6 +32,11 @@ interface CompileOptions { * The current working directory. Defaults to process.cwd() */ cwd?: string; + + /** + * `true` - only build if files do not match checksum. + */ + conditionalBuild: boolean; } export async function compile(request: CompileRequest, options?: CompileOptions): Promise { @@ -40,28 +46,55 @@ export async function compile(request: CompileRequest, options?: CompileOptions) const rootDir = path.resolve(request.rootDir || '.'); const cwd = options?.cwd; - const targetOptions: CompileTargetOptions = { + const targetOptions: CompileTargetConfig = { sort: request.sort, generateNonStrict: request.generateNonStrict, }; + const conditional = options?.conditionalBuild || false; + const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir); + + const dependencies = new Set(); for (const target of targets) { const keep = options?.filter?.(target) ?? true; if (!keep) continue; const adjustedTarget: Target = { ...targetOptions, ...target }; - await compileTarget(adjustedTarget, request, rootDir, cwd); + const deps = await compileTarget(adjustedTarget, request, { rootDir, cwd, conditional, checksumFile }); + deps.forEach((dep) => dependencies.add(dep)); + } + + if (checksumFile && dependencies.size) { + logWithTimestamp('%s', `Update checksum: ${checksumFile}`); + await updateChecksumForFiles(checksumFile, [...dependencies], { root: path.dirname(checksumFile) }); } + logWithTimestamp(`Complete.`); + + return; +} + +function resolveChecksumFile(checksumFile: string | boolean | undefined, root: string): string | undefined { + const cFilename = + (typeof checksumFile === 'string' && checksumFile) || (checksumFile && './checksum.txt') || undefined; + const file = cFilename && path.resolve(root, cFilename); + // console.warn('%o', { checksumFile, cFilename, file }); + return file; +} + +interface CompileTargetOptions { + rootDir: string; + cwd: string | undefined; + conditional: boolean; + checksumFile: string | undefined; } export async function compileTarget( target: Target, - options: CompileSourceOptions, - rootDir: string, - cwd?: string, -): Promise { + options: CompileSourceConfig, + compileOptions: CompileTargetOptions, +): Promise { logWithTimestamp(`Start compile: ${target.name}`); - + const { rootDir, cwd, checksumFile, conditional } = compileOptions; const { format, sources, trieBase, sort = true, generateNonStrict = false } = target; const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd()); @@ -79,6 +112,17 @@ export async function compileTarget( ); const filesToProcess: FileToProcess[] = await toArray(filesToProcessAsync); const normalizer = normalizeTargetWords({ sort: useTrie || sort, generateNonStrict }); + const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir; + + const deps = [...calculateDependencies(filename, filesToProcess, checksumRoot)]; + + if (conditional && checksumFile) { + const check = await checkShasumFile(checksumFile, deps, checksumRoot).catch(() => undefined); + if (check?.passed) { + logWithTimestamp(`Skip ${target.name}, nothing changed.`); + return []; + } + } const action = useTrie ? async (words: Iterable, dst: string) => { @@ -97,6 +141,24 @@ export async function compileTarget( await processFiles(action, filesToProcess, filename); logWithTimestamp(`Done compile: ${target.name}`); + + return deps; +} + +function calculateDependencies(targetFile: string, filesToProcess: FileToProcess[], rootDir: string): Set { + const dependencies = new Set(); + + addDependency(targetFile); + filesToProcess.forEach((f) => addDependency(f.src)); + + return dependencies; + + function addDependency(filename: string) { + const rel = path.relative(rootDir, filename); + dependencies.add(rel); + dependencies.add(rel.replace(/\.aff$/, '.dic')); + dependencies.add(rel.replace(/\.dic$/, '.aff')); + } } function rel(filePath: string): string { @@ -172,7 +234,7 @@ async function readFileList(fileList: FilePath): Promise { .filter((a) => !!a); } -async function readFileSource(fileSource: FileSource, sourceOptions: CompileSourceOptions): Promise { +async function readFileSource(fileSource: FileSource, sourceOptions: CompileSourceConfig): Promise { const { filename, keepRawCase = sourceOptions.keepRawCase || false, diff --git a/packages/cspell-tools/src/compiler/inlineSettings.ts b/packages/cspell-tools/src/compiler/inlineSettings.ts deleted file mode 100644 index 888acb6f3f26..000000000000 --- a/packages/cspell-tools/src/compiler/inlineSettings.ts +++ /dev/null @@ -1,43 +0,0 @@ -export interface InlineSettings { - split?: boolean; - keepRawCase?: boolean; - caseSensitive?: boolean; -} - -const allowedSettings = ['split', 'no-split', 'keep-case', 'no-keep-case', 'case-sensitive']; - -export function extractInlineSettings(line: string): InlineSettings | undefined { - const m = line.match(/cspell-tools:(.*)/); - if (!m) return undefined; - - const flags = m[1].split(/\s+/g).filter((a) => !!a); - - const settings: InlineSettings = {}; - - for (const flag of flags) { - switch (flag) { - case 'split': - settings.split = true; - break; - case 'no-split': - settings.split = false; - break; - case 'keep-case': - settings.keepRawCase = true; - break; - case 'no-keep-case': - settings.keepRawCase = false; - break; - case 'case-sensitive': - settings.caseSensitive = true; - break; - case 'flag': - case 'flags': - // Ignore flags - break; - default: - throw new Error(`Unknown inline setting: "${flag}" allowed values are ${allowedSettings.join(', ')}`); - } - } - return settings; -} diff --git a/packages/cspell-tools/src/config/config.ts b/packages/cspell-tools/src/config/config.ts index 859bf45364ed..ede99ccfacad 100644 --- a/packages/cspell-tools/src/config/config.ts +++ b/packages/cspell-tools/src/config/config.ts @@ -22,6 +22,11 @@ export interface CompileRequest extends CompileTargetOptions, CompileSourceOptio * Target Dictionaries to create. */ targets: Target[]; + + /** + * Path to checksum file. `true` - defaults to `./checksum.txt`. + */ + checksumFile?: string | boolean; } export interface Experimental { diff --git a/packages/cspell-tools/src/shasum/index.ts b/packages/cspell-tools/src/shasum/index.ts index 155d74c756a3..95ec5417d4e8 100644 --- a/packages/cspell-tools/src/shasum/index.ts +++ b/packages/cspell-tools/src/shasum/index.ts @@ -1,2 +1,2 @@ export { calcFileChecksum } from './checksum.js'; -export { checkShasumFile } from './shasum.js'; +export { checkShasumFile, updateChecksumForFiles } from './shasum.js'; diff --git a/packages/cspell-tools/src/shasum/shasum.test.ts b/packages/cspell-tools/src/shasum/shasum.test.ts index 03871f64178e..81167333b474 100644 --- a/packages/cspell-tools/src/shasum/shasum.test.ts +++ b/packages/cspell-tools/src/shasum/shasum.test.ts @@ -31,8 +31,10 @@ describe('shasum', () => { const root = resolvePathToFixture('dicts'); const filename = resolvePathToFixture('dicts/_checksum.txt'); const result = await checkShasumFile(filename, [], root); - expect(result.filter((r) => !r.passed)).toHaveLength(0); - expect(result).toMatchSnapshot(); + const results = result.results; + expect(result.passed).toBe(true); + expect(results.filter((r) => !r.passed)).toHaveLength(0); + expect(results).toMatchSnapshot(); }); test('checkShasumFile pass with files', async () => { @@ -40,9 +42,11 @@ describe('shasum', () => { const filename = resolvePathToFixture('dicts/_checksum.txt'); const files = ['colors.txt', 'cities.txt']; const result = await checkShasumFile(filename, files, root); - expect(result.filter((r) => !r.passed)).toHaveLength(0); - expect(result.map((r) => r.filename)).toEqual(files); - expect(result).toMatchSnapshot(); + const results = result.results; + expect(result.passed).toBe(true); + expect(results.filter((r) => !r.passed)).toHaveLength(0); + expect(results.map((r) => r.filename)).toEqual(files); + expect(results).toMatchSnapshot(); }); test('checkShasumFile pass with files but file not in checksum.txt', async () => { @@ -50,8 +54,10 @@ describe('shasum', () => { const filename = resolvePathToFixture('dicts/_checksum.txt'); const files = ['colors.txt', 'my_cities.txt']; const result = await checkShasumFile(filename, files, root); - expect(result.filter((r) => !r.passed)).toHaveLength(1); - expect(result.map((r) => r.filename)).toEqual(files); + const results = result.results; + expect(result.passed).toBe(false); + expect(results.filter((r) => !r.passed)).toHaveLength(1); + expect(results.map((r) => r.filename)).toEqual(files); // console.error('%o', result); }); @@ -59,8 +65,10 @@ describe('shasum', () => { const root = resolvePathToFixture('dicts'); const filename = resolvePathToFixture('dicts/_checksum-failed.txt'); const result = await checkShasumFile(filename, [], root); - expect(result.filter((r) => !r.passed)).toHaveLength(1); - expect(result).toMatchSnapshot(); + const results = result.results; + expect(result.passed).toBe(false); + expect(results.filter((r) => !r.passed)).toHaveLength(1); + expect(results).toMatchSnapshot(); }); test('checkShasumFile missing files', async () => { @@ -68,8 +76,9 @@ describe('shasum', () => { const filename = resolvePathToFixture('dicts/_checksum-missing-file.txt'); const result = await checkShasumFile(filename, [], root); // console.error('%o', result); - expect(result.filter((r) => !r.passed)).toHaveLength(1); - const missingResult = result[0]; + expect(result.passed).toBe(false); + expect(result.results.filter((r) => !r.passed)).toHaveLength(1); + const missingResult = result.results[0]; expect(missingResult.error).toEqual(Error('Failed to read file.')); }); diff --git a/packages/cspell-tools/src/shasum/shasum.ts b/packages/cspell-tools/src/shasum/shasum.ts index 9fac4fb740d2..048758bbcb05 100644 --- a/packages/cspell-tools/src/shasum/shasum.ts +++ b/packages/cspell-tools/src/shasum/shasum.ts @@ -1,11 +1,16 @@ import { readFile, writeFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { resolve, sep as pathSep } from 'node:path'; import { toError } from '../util/errors.js'; import { isDefined } from '../util/index.js'; import { calcFileChecksum, checkFile } from './checksum.js'; export interface CheckShasumFileResult { + passed: boolean; + results: CheckFileResult[]; +} + +export interface CheckFileResult { filename: string; passed: boolean; error?: Error; @@ -32,27 +37,25 @@ export async function checkShasumFile( filename: string, files: string[] | undefined, root?: string, -): Promise { +): Promise { files = !files ? files : files.length ? files : undefined; const shaFiles = await readAndParseShasumFile(filename); const filesToCheck = !files ? shaFiles.map(({ filename }) => filename) : files; - const mapNameToChecksum = new Map(shaFiles.map((r) => [r.filename, r.checksum] as const)); + const mapNameToChecksum = new Map(shaFiles.map((r) => [normalizeFilename(r.filename), r.checksum] as const)); const resolvedRoot = resolve(root || '.'); - const results: CheckShasumFileResult[] = await Promise.all( - filesToCheck.map((filename) => { + const results: CheckFileResult[] = await Promise.all( + filesToCheck.map(normalizeFilename).map((filename) => { return tryToCheckFile(filename, resolvedRoot, mapNameToChecksum.get(filename)); }), ); - return results; + const passed = !results.find((v) => !v.passed); + + return { passed, results }; } -async function tryToCheckFile( - filename: string, - root: string, - checksum: string | undefined, -): Promise { +async function tryToCheckFile(filename: string, root: string, checksum: string | undefined): Promise { if (!checksum) { return { filename, passed: false, error: Error('Missing Checksum.') }; } @@ -135,11 +138,12 @@ export async function reportCheckChecksumFile( ): Promise { const root = options.root; const filesToCheck = await resolveFileList(files, options.listFile); - const result = await checkShasumFile(filename, filesToCheck, root); - const lines = result.map(({ filename, passed, error }) => + const checkResult = await checkShasumFile(filename, filesToCheck, root); + const results = checkResult.results; + const lines = results.map(({ filename, passed, error }) => `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim(), ); - const withErrors = result.filter((a) => !a.passed); + const withErrors = results.filter((a) => !a.passed); const passed = !withErrors.length; if (!passed) { lines.push( @@ -164,7 +168,7 @@ async function resolveFileList(files: string[] | undefined, listFile: string[] | .filter((a) => a) .forEach((file) => setOfFiles.add(file)); } - return [...setOfFiles]; + return [...setOfFiles].map(normalizeFilename); } export async function calcUpdateChecksumForFiles( @@ -174,11 +178,13 @@ export async function calcUpdateChecksumForFiles( ): Promise { const root = options.root || '.'; const filesToCheck = await resolveFileList(files, options.listFile); - const currentEntries = await readAndParseShasumFile(filename).catch((err) => { - const e = toError(err); - if (e.code !== 'ENOENT') throw e; - return [] as ChecksumEntry[]; - }); + const currentEntries = ( + await readAndParseShasumFile(filename).catch((err) => { + const e = toError(err); + if (e.code !== 'ENOENT') throw e; + return [] as ChecksumEntry[]; + }) + ).map((entry) => ({ ...entry, filename: normalizeFilename(entry.filename) })); const entriesToUpdate = new Set([...filesToCheck, ...currentEntries.map((e) => e.filename)]); const mustExist = new Set(filesToCheck); @@ -211,3 +217,7 @@ export async function updateChecksumForFiles( return { passed: true, report: content }; } + +function normalizeFilename(filename: string): string { + return filename.split(pathSep).join('/'); +} diff --git a/vitest.config.ts b/vitest.config.ts index 78315d822275..2bc55d92bdc4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ clean: true, all: true, reportsDirectory: 'coverage', - reporter: ['html', 'text', 'json'], + reporter: ['html', 'json', ['lcov', { projectRoot: __dirname }], 'text'], exclude: [ 'ajv.config.*', 'bin.mjs',