fix: Improve fetching/caching config files

integration-tests/config/config.json

@@ -323,7 +323,9 @@
       "path": "eslint/eslint",
       "url": "https://github.com/eslint/eslint",
       "args": [
-        "**",
+        ".",
+        "--config=../../../config/eslint/cspell.config.yaml",
+        "--issues-summary-report",
         "--exclude=bin/**",
         "--exclude=CHANGELOG.md",
         "--exclude=_data",

integration-tests/repositories/config/eslint/cspell.config.yaml

@@ -0,0 +1,78 @@
+import:
+  - ../../cspell.yaml
+words:
+  - autofix
+  - autofixes
+  - autofixer
+  - autofixed
+  - autofixable
+  - autofixing
+  - backreferences
+  - backreference
+  - builtins
+  - camelcase
+  - camelcased
+  - checkstyle
+  - classname
+  - codebase
+  - codebases
+  - codeframe
+  - dosomething
+  - doublecircle
+  - doublequote
+  - eqeqeq
+  - eslintconfig
+  - eslintplugin
+  - eslintignore
+  - espree
+  - escope
+  - esquery
+  - eslintrc
+  - eslintrcjson
+  - eslintrcyaml
+  - eslintrcjs
+  - eslintrcjson
+  - eslintrcyaml
+  - eslintrcjs
+  - fileoverview
+  - fillcolor
+  - fixedsize
+  - fixedmode
+  - gitignores
+  - globalstrict
+  - hashbang
+  - hashbangs
+  - iife
+  - iifes
+  - imurmurhash
+  - instanceof
+  - linebreak
+  - linebreaks
+  - autocrlf
+  - concat
+  - hiddenfolder
+  - hola
+  - lookaround
+  - lookbehinds
+  - multipass
+  - parenthesised
+  - pluggable
+  - postprocess
+  - postprocessed
+  - preprocess
+  - redeclaration
+  - redeclarations
+  - redeclared
+  - regexpp
+  - rulesdir
+  - rulesdirs
+  - singlequote
+  - testcase
+  - testcases
+  - testsuite
+  - testsuites
+  - typedefs
+  - unignore
+  - unignored
+  - unignores
+  - unparenthesized

integration-tests/src/reporter/index.ts

@@ -1,9 +1,10 @@
-import type { CSpellReporter, Issue, RunResult } from '@cspell/cspell-types';
+import type { CSpellReporter, Issue, ReporterConfiguration, RunResult } from '@cspell/cspell-types';
 import * as vscodeUri from 'vscode-uri';
 
 import { readConfig } from '../config';
 import type { Repository } from '../configDef';
 import { writeSnapshotRaw } from '../snapshots';
+import type { IssueSummary } from './reportGenerator';
 import { generateReport } from './reportGenerator';
 import { stringify } from './stringify';
 
@@ -13,10 +14,18 @@ const noopReporter = () => {
     return;
 };
 
-export function getReporter(): CSpellReporter {
+interface Config extends ReporterConfiguration {
+    issuesSummaryReport?: boolean;
+}
+
+export function getReporter(_settings: unknown, config?: Config): CSpellReporter {
+    const issueFilter = config?.unique ? uniqueFilter((i: Issue) => i.text) : () => true;
     const issues: Issue[] = [];
     const errors: string[] = [];
     const files: string[] = [];
+    const issuesSummaryReport = !!config?.issuesSummaryReport;
+    const issuesSummary = new Map<string, IssueSummary>();
+    const summaryAccumulator = createIssuesSummaryAccumulator(issuesSummary);
 
     async function processResult(result: RunResult): Promise<void> {
         const root = URI.file(process.cwd());
@@ -27,14 +36,18 @@ export function getReporter(): CSpellReporter {
             runResult: result,
             root,
             repository: fetchRepositoryInfo(root),
+            issuesSummary: issuesSummaryReport && issuesSummary.size ? [...issuesSummary.values()] : undefined,
         });
         const repPath = extractRepositoryPath(root);
         writeSnapshotRaw(repPath, 'report.yaml', stringify(report));
     }
 
     const reporter: CSpellReporter = {
         issue: (issue) => {
-            issues.push(issue);
+            summaryAccumulator(issue);
+            if (issueFilter(issue)) {
+                issues.push(issue);
+            }
         },
         info: noopReporter,
         debug: noopReporter,
@@ -60,3 +73,33 @@ function fetchRepositoryInfo(root: vscodeUri.URI): Repository | undefined {
     const path = extractRepositoryPath(root);
     return reps.get(path);
 }
+
+function uniqueFilter<T, K>(keyFn: (v: T) => K): (v: T) => boolean {
+    const seen = new Set<K>();
+    return (v) => {
+        const k = keyFn(v);
+        if (seen.has(k)) return false;
+        seen.add(k);
+        return true;
+    };
+}
+
+function createIssuesSummaryAccumulator(issuesSummary: Map<string, IssueSummary>): (issue: Issue) => void {
+    function uniqueKey(issue: Issue): string {
+        return [issue.text, issue.uri || ''].join('::');
+    }
+
+    const isUnique = uniqueFilter(uniqueKey);
+
+    return (issue: Issue) => {
+        const { text } = issue;
+        const summary = issuesSummary.get(text) || { text, count: 0, files: 0 };
+        const unique = isUnique(issue);
+        summary.count += 1;
+        summary.files += unique ? 1 : 0;
+        if (issue.isFlagged) {
+            summary.isFlagged = true;
+        }
+        issuesSummary.set(text, summary);
+    };
+}

integration-tests/src/reporter/reportGenerator.ts

@@ -8,6 +8,7 @@ export type Report = {
     errors: string[];
     summary: Summary;
     repository: Repository | undefined;
+    issuesSummary: IssueSummary[];
 };
 
 export interface Summary {
@@ -17,6 +18,13 @@ export interface Summary {
     errors: number;
 }
 
+export interface IssueSummary {
+    text: string;
+    isFlagged?: boolean | undefined;
+    count: number;
+    files: number;
+}
+
 type SortedFileIssues = FileIssue[];
 type FileIssue = string;
 
@@ -27,6 +35,7 @@ export interface ReportData {
     root: Uri;
     runResult: RunResult;
     repository: Repository | undefined;
+    issuesSummary?: IssueSummary[] | undefined;
 }
 
 const compare = new Intl.Collator().compare;
@@ -61,6 +70,7 @@ export function generateReport(data: ReportData): Report {
 
     const base: SortedFileIssues = [];
     const fileIssues: SortedFileIssues = base.concat(...issuesByFile);
+    const issuesSummary = [...(data.issuesSummary || [])].sort((a, b) => compare(a.text, b.text));
 
     return {
         fileIssues,
@@ -72,6 +82,7 @@ export function generateReport(data: ReportData): Report {
             errors: runResult.errors,
         },
         repository: data.repository,
+        issuesSummary,
     };
 }
 

integration-tests/src/reporter/stringify.ts

@@ -29,5 +29,17 @@ export function stringify(report: Report): string {
         defaultKeyType: 'PLAIN',
         doubleQuotedAsJSON: true,
     });
-    return [header, issues].join('\n');
+
+    const issuesSummary = report.issuesSummary?.length
+        ? stringifyYaml(
+              {
+                  issuesSummary: report.issuesSummary.map((issue) =>
+                      stringifyYaml(issue).replace(/\n/g, ', ').replace(/\s+/g, ' ').trim(),
+                  ),
+              },
+              { lineWidth: 200 },
+          )
+        : '';
+
+    return [header, issues, issuesSummary].filter((a) => !!a).join('\n');
 }

packages/cspell-eslint-plugin/src/worker/spellCheck.mts

@@ -351,7 +351,7 @@ function getDocValidator(filename: string, text: string, options: WorkerOptions)
     const cachedValidator = docValCache.get(doc);
     if (cachedValidator && deepEqual(cachedValidator.settings, settings)) {
         refreshDictionaryCache(0);
-        cachedValidator.updateDocumentText(text);
+        cachedValidator.updateDocumentText(text).catch(() => undefined);
         return cachedValidator;
     }
 

packages/cspell-lib/api/api.d.ts

@@ -388,9 +388,17 @@ declare function searchForConfig(searchFrom: URL | string | undefined, pnpSettin
  * @returns normalized CSpellSettings
  */
 declare function loadConfig(file: string, pnpSettings?: PnPSettingsOptional): Promise<CSpellSettingsI$1>;
+/**
+ * Might throw if the settings have not yet been loaded.
+ * @deprecated use {@link getGlobalSettingsAsync} instead.
+ */
 declare function getGlobalSettings(): CSpellSettingsI$1;
+/**
+ * Loads and caches the global settings.
+ * @returns - global settings
+ */
+declare function getGlobalSettingsAsync(): Promise<CSpellSettingsI$1>;
 declare function getCachedFileSize(): number;
-declare function clearCachedSettingsFiles(): void;
 declare function readRawSettings(filename: string | URL, relativeTo?: string | URL): Promise<CSpellSettingsWST$1>;
 
 declare function extractImportErrors(settings: CSpellSettingsWST$1): ImportFileRefWithError$1[];
@@ -636,6 +644,11 @@ interface ValidateTextOptions {
      * Verify that the in-document directives are correct.
      */
     validateDirectives?: boolean;
+    /**
+     * Skips spell checking the document. Useful for testing and dry runs.
+     * It will read the configuration and parse the document.
+     */
+    skipValidation?: boolean;
 }
 
 interface DocumentValidatorOptions extends ValidateTextOptions {
@@ -654,6 +667,7 @@ interface DocumentValidatorOptions extends ValidateTextOptions {
      */
     noConfigSearch?: boolean;
 }
+type PerfTimings = Record<string, number>;
 declare class DocumentValidator {
     readonly settings: CSpellUserSettings;
     private _document;
@@ -664,6 +678,8 @@ declare class DocumentValidator {
     private _preparationTime;
     private _suggestions;
     readonly options: DocumentValidatorOptions;
+    readonly perfTiming: PerfTimings;
+    skipValidation: boolean;
     /**
      * @param doc - Document to validate
      * @param config - configuration to use (not finalized).
@@ -717,6 +733,8 @@ declare class DocumentValidator {
      * Internal `cspell-lib` use.
      */
     _getPreparations(): Preparations | undefined;
+    private static getGlobMatcher;
+    private static _getGlobMatcher;
 }
 interface Preparations {
     /** loaded config */
@@ -800,6 +818,12 @@ interface SpellCheckFileOptions extends ValidateTextOptions {
      */
     noConfigSearch?: boolean;
 }
+interface SpellCheckFilePerf extends Record<string, number | undefined> {
+    loadTimeMs?: number;
+    prepareTimeMs?: number;
+    checkTimeMs?: number;
+    totalTimeMs?: number;
+}
 interface SpellCheckFileResult {
     document: Document | DocumentWithText;
     settingsUsed: CSpellSettingsWithSourceTrace;
@@ -808,6 +832,7 @@ interface SpellCheckFileResult {
     issues: ValidationIssue[];
     checked: boolean;
     errors: Error[] | undefined;
+    perf?: SpellCheckFilePerf;
 }
 /**
  * Spell Check a file
@@ -893,7 +918,21 @@ interface ResolveFileResult {
  */
 declare function resolveFile(filename: string, relativeTo: string | URL): ResolveFileResult;
 
+/**
+ * Clear the cached files and other cached data.
+ * Calling this function will cause the next spell check to take longer because it will need to reload configuration files and dictionaries.
+ * Call this function if configuration files have changed.
+ *
+ * It is safe to replace {@link clearCachedFiles} with {@link clearCaches}
+ */
 declare function clearCachedFiles(): Promise<void>;
+/**
+ * Sends and event to clear the caches.
+ * It resets the configuration files and dictionaries.
+ *
+ * It is safe to replace {@link clearCaches} with {@link clearCachedFiles}
+ */
+declare function clearCaches(): void;
 
 /**
  * Load a dictionary collection defined by the settings.
@@ -902,4 +941,14 @@ declare function clearCachedFiles(): Promise<void>;
  */
 declare function getDictionary(settings: CSpellUserSettings): Promise<SpellingDictionaryCollection>;
 
-export { type CheckTextInfo, type ConfigurationDependencies, type CreateTextDocumentParams, type DetermineFinalDocumentSettingsResult, type Document, DocumentValidator, type DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, type ExcludeFilesGlobMap, type ExclusionFunction, exclusionHelper_d as ExclusionHelper, type FeatureFlag, FeatureFlags, ImportError, type ImportFileRefWithError, IncludeExcludeFlag, type IncludeExcludeOptions, index_link_d as Link, type Logger, type SpellCheckFileOptions, type SpellCheckFileResult, SpellingDictionaryLoadError, type SuggestedWord, SuggestionError, type SuggestionOptions, type SuggestionsForWordResult, text_d as Text, type TextDocument, type TextDocumentLine, type TextDocumentRef, type TextInfoItem, type TraceOptions, type TraceResult, UnknownFeatureFlagError, type ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, checkTextDocument, clearCachedFiles, clearCachedSettingsFiles, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, fileToTextDocument, finalizeSettings, getCachedFileSize, getDefaultBundledSettingsAsync, getDefaultSettings, getDictionary, getGlobalSettings, getLanguagesForBasename as getLanguageIdsForBaseFilename, getLanguagesForExt, getLogger, getSources, getSystemFeatureFlags, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, loadPnPSync, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, shouldCheckDocument, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText };
+interface PerfTimer {
+    readonly name: string;
+    readonly startTime: number;
+    readonly elapsed: number;
+    start(): void;
+    end(): void;
+}
+type TimeNowFn = () => number;
+declare function createPerfTimer(name: string, onEnd?: (elapsed: number, name: string) => void, timeNowFn?: TimeNowFn): PerfTimer;
+
+export { type CheckTextInfo, type ConfigurationDependencies, type CreateTextDocumentParams, type DetermineFinalDocumentSettingsResult, type Document, DocumentValidator, type DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, type ExcludeFilesGlobMap, type ExclusionFunction, exclusionHelper_d as ExclusionHelper, type FeatureFlag, FeatureFlags, ImportError, type ImportFileRefWithError, IncludeExcludeFlag, type IncludeExcludeOptions, index_link_d as Link, type Logger, type PerfTimer, type SpellCheckFileOptions, type SpellCheckFileResult, SpellingDictionaryLoadError, type SuggestedWord, SuggestionError, type SuggestionOptions, type SuggestionsForWordResult, text_d as Text, type TextDocument, type TextDocumentLine, type TextDocumentRef, type TextInfoItem, type TraceOptions, type TraceResult, UnknownFeatureFlagError, type ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, checkTextDocument, clearCachedFiles, clearCaches, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createPerfTimer, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, fileToTextDocument, finalizeSettings, getCachedFileSize, getDefaultBundledSettingsAsync, getDefaultSettings, getDictionary, getGlobalSettings, getGlobalSettingsAsync, getLanguagesForBasename as getLanguageIdsForBaseFilename, getLanguagesForExt, getLogger, getSources, getSystemFeatureFlags, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, loadPnPSync, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, shouldCheckDocument, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText };

packages/cspell-lib/src/lib/Settings/Controller/configLoader/configLoader.test.ts

@@ -23,6 +23,7 @@ import {
     getCachedFileSize,
     getDefaultConfigLoader,
     getGlobalSettings,
+    getGlobalSettingsAsync,
     loadConfig,
     readConfigFile,
     readRawSettings,
@@ -109,13 +110,14 @@ describe('Validate CSpellSettingsServer', () => {
 
     test('makes sure global settings is an object', async () => {
         const settings = getGlobalSettings();
+        expect(await getGlobalSettingsAsync()).toBe(settings);
         expect(Object.keys(settings)).not.toHaveLength(0);
-        const merged = mergeSettings(await getDefaultBundledSettingsAsync(), await getGlobalSettings());
+        const merged = mergeSettings(await getDefaultBundledSettingsAsync(), await getGlobalSettingsAsync());
         expect(Object.keys(merged)).not.toHaveLength(0);
     });
 
     test('verify clearing the file cache works', async () => {
-        mergeSettings(await getDefaultBundledSettingsAsync(), await getGlobalSettings());
+        mergeSettings(await getDefaultBundledSettingsAsync(), await getGlobalSettingsAsync());
         expect(getCachedFileSize()).toBeGreaterThan(0);
         clearCachedSettingsFiles();
         expect(getCachedFileSize()).toBe(0);