From 7518c72e2382cb9a2fc583fd17dfc307aea49fe1 Mon Sep 17 00:00:00 2001 From: mattcompiles Date: Thu, 14 Mar 2024 15:27:43 +1100 Subject: [PATCH 1/2] Add bailout timeout to respondToFsEvents --- packages/core/core/src/Parcel.js | 7 +- packages/core/core/src/RequestTracker.js | 95 ++++++++++++++++++---- packages/core/core/src/types.js | 7 +- packages/core/diagnostic/src/diagnostic.js | 15 ++++ packages/core/types/index.js | 4 +- 5 files changed, 102 insertions(+), 26 deletions(-) diff --git a/packages/core/core/src/Parcel.js b/packages/core/core/src/Parcel.js index 24b25a91a17..3d94666ce82 100644 --- a/packages/core/core/src/Parcel.js +++ b/packages/core/core/src/Parcel.js @@ -426,10 +426,9 @@ export default class Parcel { } let isInvalid = this.#requestTracker.respondToFSEvents( - events.map(e => ({ - type: e.type, - path: toProjectPath(resolvedOptions.projectRoot, e.path), - })), + events, + resolvedOptions.projectRoot, + Number.POSITIVE_INFINITY, ); if (isInvalid && this.#watchQueue.getNumWaiting() === 0) { if (this.#watchAbortController) { diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index c964e00292a..d2c075e37e2 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -2,7 +2,7 @@ import type {AbortSignal} from 'abortcontroller-polyfill/dist/cjs-ponyfill'; import type {Async, EnvMap} from '@parcel/types'; -import type {EventType, Options as WatcherOptions} from '@parcel/watcher'; +import type {Options as WatcherOptions, Event} from '@parcel/watcher'; import type WorkerFarm from '@parcel/workers'; import type { ContentGraphOpts, @@ -750,10 +750,37 @@ export class RequestGraph extends ContentGraph< } respondToFSEvents( - events: Array<{|path: ProjectPath, type: EventType|}>, + events: Array, + projectRoot: string, + threshold: number, ): boolean { let didInvalidate = false; - for (let {path: _filePath, type} of events) { + let count = 0; + let predictedTime = 0; + let startTime = Date.now(); + + for (let {path: _path, type} of events) { + if (++count === 256) { + let duration = Date.now() - startTime; + predictedTime = duration * (events.length >> 8); + if (predictedTime > threshold) { + logger.warn({ + origin: '@parcel/core', + message: + 'Building with clean cache. Cache invalidation took too long.', + meta: { + trackableEvent: 'cache_invalidation_timeout', + watcherEventCount: events.length, + predictedTime, + }, + }); + throw new Error( + 'Responding to file system events exceeded threshold, start with empty cache.', + ); + } + } + + let _filePath = toProjectPath(projectRoot, _path); let filePath = fromProjectPathRelative(_filePath); let hasFileRequest = this.hasContentKey(filePath); @@ -761,12 +788,13 @@ export class RequestGraph extends ContentGraph< // this means the project root was moved and we need to // re-run all requests. if (type === 'create' && filePath === '') { - // $FlowFixMe(incompatible-call) `trackableEvent` isn't part of the Diagnostic interface logger.verbose({ origin: '@parcel/core', message: 'Watcher reported project root create event. Invalidate all nodes.', - trackableEvent: 'project_root_create', + meta: { + trackableEvent: 'project_root_create', + }, }); for (let [id, node] of this.nodes.entries()) { if (node?.type === REQUEST) { @@ -860,6 +888,17 @@ export class RequestGraph extends ContentGraph< } } + let duration = Date.now() - startTime; + logger.verbose({ + origin: '@parcel/core', + message: `RequestGraph.respondToFSEvents duration: ${duration}`, + meta: { + trackableEvent: 'fsevent_response_time', + duration, + predictedTime, + }, + }); + return didInvalidate && this.invalidNodeIds.size > 0; } @@ -994,9 +1033,11 @@ export default class RequestTracker { } respondToFSEvents( - events: Array<{|path: ProjectPath, type: EventType|}>, + events: Array, + projectRoot: string, + threshold: number, ): boolean { - return this.graph.respondToFSEvents(events); + return this.graph.respondToFSEvents(events, projectRoot, threshold); } hasInvalidRequests(): boolean { @@ -1363,24 +1404,48 @@ async function loadRequestGraph(options): Async { let opts = getWatcherOptions(options); let snapshotKey = `snapshot-${cacheKey}`; let snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt'); + + let timeout = setTimeout(() => { + logger.warn({ + origin: '@parcel/core', + message: `Retrieving file system events since last build...\nThis can take upto a minute after branch changes or npm/yarn installs.`, + }); + }, 5000); + let startTime = Date.now(); let events = await options.inputFS.getEventsSince( - options.watchDir, + options.projectRoot, snapshotPath, opts, ); + clearTimeout(timeout); + + logger.verbose({ + origin: '@parcel/core', + message: `File system event count: ${events.length}`, + meta: { + trackableEvent: 'watcher_events_count', + watcherEventCount: events.length, + duration: Date.now() - startTime, + }, + }); requestGraph.invalidateUnpredictableNodes(); requestGraph.invalidateOnBuildNodes(); requestGraph.invalidateEnvNodes(options.env); requestGraph.invalidateOptionNodes(options); - requestGraph.respondToFSEvents( - (options.unstableFileInvalidations || events).map(e => ({ - type: e.type, - path: toProjectPath(options.projectRoot, e.path), - })), - ); - return requestGraph; + try { + requestGraph.respondToFSEvents( + options.unstableFileInvalidations || events, + options.projectRoot, + 10000, + ); + return requestGraph; + } catch (e) { + // This error means respondToFSEvents timed out handling the invalidation events + // In this case we'll return a fresh RequestGraph + return new RequestGraph(); + } } return new RequestGraph(); diff --git a/packages/core/core/src/types.js b/packages/core/core/src/types.js index eef7f20608d..578ca61fa74 100644 --- a/packages/core/core/src/types.js +++ b/packages/core/core/src/types.js @@ -31,7 +31,7 @@ import type {FileSystem} from '@parcel/fs'; import type {Cache} from '@parcel/cache'; import type {PackageManager} from '@parcel/package-manager'; import type {ProjectPath} from './projectPath'; -import type {EventType} from '@parcel/watcher'; +import type {Event} from '@parcel/watcher'; import type {FeatureFlags} from '@parcel/feature-flags'; import type {BackendType} from '@parcel/watcher'; @@ -295,10 +295,7 @@ export type ParcelOptions = {| shouldTrace: boolean, shouldPatchConsole: boolean, detailedReport?: ?DetailedReportOptions, - unstableFileInvalidations?: Array<{| - path: FilePath, - type: EventType, - |}>, + unstableFileInvalidations?: Array, inputFS: FileSystem, outputFS: FileSystem, diff --git a/packages/core/diagnostic/src/diagnostic.js b/packages/core/diagnostic/src/diagnostic.js index ca17cc06253..272408db30c 100644 --- a/packages/core/diagnostic/src/diagnostic.js +++ b/packages/core/diagnostic/src/diagnostic.js @@ -46,6 +46,18 @@ export type DiagnosticCodeFrame = {| codeHighlights: Array, |}; +type JSONValue = + | null + | void // ? Is this okay? + | boolean + | number + | string + | Array + | JSONObject; + +/** A JSON object (as in "map") */ +type JSONObject = {[key: string]: JSONValue, ...}; + /** * A style agnostic way of emitting errors, warnings and info. * Reporters are responsible for rendering the message, codeframes, hints, ... @@ -72,6 +84,9 @@ export type Diagnostic = {| /** A URL to documentation to learn more about the diagnostic. */ documentationURL?: string, + + /** Diagnostic specific metadata (optional) */ + meta?: JSONObject, |}; // This type should represent all error formats Parcel can encounter... diff --git a/packages/core/types/index.js b/packages/core/types/index.js index 54ae3a7f050..f547e88a100 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -15,7 +15,7 @@ import type {Cache} from '@parcel/cache'; import type {AST as _AST, ConfigResult as _ConfigResult} from './unsafe'; import type {TraceMeasurement} from '@parcel/profiler'; import type {FeatureFlags} from '@parcel/feature-flags'; -import type {EventType, BackendType} from '@parcel/watcher'; +import type {Event, BackendType} from '@parcel/watcher'; /** Plugin-specific AST, any */ export type AST = _AST; @@ -313,7 +313,7 @@ export type InitialParcelOptions = {| +lazyIncludes?: string[], +lazyExcludes?: string[], +shouldBundleIncrementally?: boolean, - +unstableFileInvalidations?: Array<{|path: FilePath, type: EventType|}>, + +unstableFileInvalidations?: Array, +inputFS?: FileSystem, +outputFS?: FileSystem, From e8eeaa62a93a01ad2db3c920d10929d4f1aaf373 Mon Sep 17 00:00:00 2001 From: mattcompiles Date: Thu, 14 Mar 2024 15:43:23 +1100 Subject: [PATCH 2/2] Fix getEventsSince --- packages/core/core/src/RequestTracker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index d2c075e37e2..efb6ef42077 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -1413,7 +1413,7 @@ async function loadRequestGraph(options): Async { }, 5000); let startTime = Date.now(); let events = await options.inputFS.getEventsSince( - options.projectRoot, + options.watchDir, snapshotPath, opts, );