diff --git a/docs/generated/devkit/FileData.md b/docs/generated/devkit/FileData.md index d56c3ebad6cf22..bdb6b8f2c5b681 100644 --- a/docs/generated/devkit/FileData.md +++ b/docs/generated/devkit/FileData.md @@ -6,7 +6,7 @@ Some metadata about a file ### Properties -- [deps](../../devkit/documents/FileData#deps): (string | [string, string])[] +- [deps](../../devkit/documents/FileData#deps): FileDataDependency[] - [file](../../devkit/documents/FileData#file): string - [hash](../../devkit/documents/FileData#hash): string @@ -14,7 +14,13 @@ Some metadata about a file ### deps -• `Optional` **deps**: (`string` \| [`string`, `string`])[] +• `Optional` **deps**: `FileDataDependency`[] + +An array of dependencies. If an element is just a string, +the dependency is assumed to be a static dependency targetting +that string. If the element is a tuple with two elements, the first element +inside of it is the target project, with the second element being the type of dependency. +If the tuple has 3 elements, the first is preceded by a source. --- diff --git a/docs/generated/devkit/ProjectGraphBuilder.md b/docs/generated/devkit/ProjectGraphBuilder.md index 7e0669d961a8fe..bb82d6e8814a2a 100644 --- a/docs/generated/devkit/ProjectGraphBuilder.md +++ b/docs/generated/devkit/ProjectGraphBuilder.md @@ -14,7 +14,9 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ### Properties +- [allWorkspaceFiles](../../devkit/documents/ProjectGraphBuilder#allworkspacefiles): FileData[] - [graph](../../devkit/documents/ProjectGraphBuilder#graph): ProjectGraph +- [nonProjectFiles](../../devkit/documents/ProjectGraphBuilder#nonprojectfiles): FileData[] - [projectFileMap](../../devkit/documents/ProjectGraphBuilder#projectfilemap): ProjectFileMap - [removedEdges](../../devkit/documents/ProjectGraphBuilder#removededges): Object @@ -40,23 +42,36 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ### constructor -• **new ProjectGraphBuilder**(`graph?`, `fileMap?`) +• **new ProjectGraphBuilder**(`graph?`, `projectFileMap?`, `nonProjectFiles?`) #### Parameters -| Name | Type | -| :--------- | :-------------------------------------------------------- | -| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) | -| `fileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) | +| Name | Type | +| :----------------- | :-------------------------------------------------------- | +| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) | +| `projectFileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) | +| `nonProjectFiles?` | [`FileData`](../../devkit/documents/FileData)[] | ## Properties +### allWorkspaceFiles + +• `Private` `Readonly` **allWorkspaceFiles**: [`FileData`](../../devkit/documents/FileData)[] + +--- + ### graph • `Readonly` **graph**: [`ProjectGraph`](../../devkit/documents/ProjectGraph) --- +### nonProjectFiles + +• `Private` `Readonly` **nonProjectFiles**: [`FileData`](../../devkit/documents/FileData)[] + +--- + ### projectFileMap • `Private` `Readonly` **projectFileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) diff --git a/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts b/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts index e9b138255245c3..0697b000b59ef4 100644 --- a/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts +++ b/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts @@ -9,7 +9,7 @@ import type { ImportDeclaration, ImportSpecifier, Node } from 'typescript'; import { FileChangeRecorder } from '../../utils/file-change-recorder'; import { ngrxVersion } from '../../utils/versions'; import { getProjectsFilteredByDependencies } from '../utils/projects'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; import { fileDataDepTarget } from 'nx/src/config/project-graph'; let tsquery: typeof import('@phenomnomnominal/tsquery').tsquery; @@ -35,7 +35,7 @@ export default async function (tree: Tree): Promise { ensureTypescript(); tsquery = require('@phenomnomnominal/tsquery').tsquery; - const cachedFileMap = readProjectFileMapCache().projectFileMap; + const cachedFileMap = readFileMapCache().fileMap.projectFileMap; const filesWithNxAngularImports: FileData[] = []; for (const { graphNode } of projects) { diff --git a/packages/eslint-plugin/src/utils/graph-utils.ts b/packages/eslint-plugin/src/utils/graph-utils.ts index 063bd96e7774a4..bb4ef5107f192d 100644 --- a/packages/eslint-plugin/src/utils/graph-utils.ts +++ b/packages/eslint-plugin/src/utils/graph-utils.ts @@ -149,7 +149,7 @@ export function findFilesInCircularPath( for (let i = 0; i < circularPath.length - 1; i++) { const next = circularPath[i + 1].name; - const files: FileData[] = projectFileMap[circularPath[i].name] || []; + const files = projectFileMap[circularPath[i].name] || []; filePathChain.push( files .filter( diff --git a/packages/eslint-plugin/src/utils/project-graph-utils.ts b/packages/eslint-plugin/src/utils/project-graph-utils.ts index b7ffe82af53291..27afc607e85d1c 100644 --- a/packages/eslint-plugin/src/utils/project-graph-utils.ts +++ b/packages/eslint-plugin/src/utils/project-graph-utils.ts @@ -11,7 +11,7 @@ import { } from 'nx/src/project-graph/utils/find-project-for-path'; import { readNxJson } from 'nx/src/config/configuration'; import { TargetProjectLocator } from '@nx/js/src/internal'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; export function ensureGlobalProjectGraph(ruleName: string) { /** @@ -37,7 +37,7 @@ export function ensureGlobalProjectGraph(ruleName: string) { globalThis.projectRootMappings = createProjectRootMappings( projectGraph.nodes ); - globalThis.projectFileMap = readProjectFileMapCache().projectFileMap; + globalThis.projectFileMap = readFileMapCache().fileMap.projectFileMap; globalThis.targetProjectLocator = new TargetProjectLocator( projectGraph.nodes, projectGraph.externalNodes diff --git a/packages/js/src/utils/package-json/update-package-json.ts b/packages/js/src/utils/package-json/update-package-json.ts index 74cd9cb0f2fe11..41630fc895be20 100644 --- a/packages/js/src/utils/package-json/update-package-json.ts +++ b/packages/js/src/utils/package-json/update-package-json.ts @@ -25,7 +25,7 @@ import { writeFileSync } from 'fs-extra'; import { fileExists } from 'nx/src/utils/fileutils'; import type { PackageJson } from 'nx/src/utils/package-json'; import { existsSync } from 'fs'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; import { getRelativeDirectoryToProjectRoot } from '../get-main-file-dir'; @@ -57,7 +57,7 @@ export function updatePackageJson( ): void { let packageJson: PackageJson; if (fileMap == null) { - fileMap = readProjectFileMapCache()?.projectFileMap || {}; + fileMap = readFileMapCache()?.fileMap?.projectFileMap || {}; } if (options.updateBuildableProjectDepsInPackageJson) { diff --git a/packages/nx/src/config/project-graph.ts b/packages/nx/src/config/project-graph.ts index 14b90a7e1f65bc..b6f8a54dbe2560 100644 --- a/packages/nx/src/config/project-graph.ts +++ b/packages/nx/src/config/project-graph.ts @@ -11,15 +11,42 @@ import { NxJsonConfiguration } from './nx-json'; export interface FileData { file: string; hash: string; - deps?: (string | [string, string])[]; + /** + * An array of dependencies. If an element is just a string, + * the dependency is assumed to be a static dependency targetting + * that string. If the element is a tuple with two elements, the first element + * inside of it is the target project, with the second element being the type of dependency. + * If the tuple has 3 elements, the first is preceded by a source. + */ + deps?: FileDataDependency[]; } -export function fileDataDepTarget(dep: string | [string, string]) { - return typeof dep === 'string' ? dep : dep[0]; +/** + * A file data dependency, as stored in the cache. If just a string, + * the dependency is assumed to be a static dependency targetting + * that string. If it is a tuple with two elements, the first element + * inside of it is the target project, with the second element being the type of dependency. + * If the tuple has 3 elements, the first is preceded by a source. + */ +export type FileDataDependency = + | string + | [target: string, type: DependencyType] + | [source: string, target: string, type: DependencyType]; + +export function fileDataDepTarget(dep: FileDataDependency) { + return typeof dep === 'string' + ? dep + : Array.isArray(dep) && dep.length === 2 + ? dep[0] + : dep[1]; } -export function fileDataDepType(dep: string | [string, string]) { - return typeof dep === 'string' ? 'static' : dep[1]; +export function fileDataDepType(dep: FileDataDependency) { + return typeof dep === 'string' + ? 'static' + : Array.isArray(dep) && dep.length === 2 + ? dep[0] + : dep[1]; } export interface FileMap { diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index fb6bf04b75a165..915361d631ad34 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -247,7 +247,7 @@ async function processFilesAndCreateAndSerializeProjectGraph() { } } -function copyFileData(d: FileData[]) { +function copyFileData(d: T[]) { return d.map((t) => ({ ...t })); } @@ -257,7 +257,7 @@ function copyFileMap(m: FileMap) { projectFileMap: {}, }; for (let p of Object.keys(m.projectFileMap)) { - c[p] = copyFileData(m.projectFileMap[p]); + c.projectFileMap[p] = copyFileData(m.projectFileMap[p]); } return c; } diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 36fc2a3fb22562..1cacc9b6025c4f 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -4,6 +4,7 @@ import { performance } from 'perf_hooks'; import { assertWorkspaceValidity } from '../utils/assert-workspace-validity'; import { FileData } from './file-utils'; import { + CachedFileData, createProjectFileMapCache, extractCachedFileData, FileMapCache, @@ -78,10 +79,10 @@ export async function buildProjectGraphUsingProjectFileMap( const packageJsonDeps = readCombinedDeps(); const rootTsConfig = readRootTsConfig(); - let filesToProcess; - let cachedFileData; + let filesToProcess: FileMap; + let cachedFileData: CachedFileData; const useCacheData = - fileMap && + fileMapCache && !shouldRecomputeWholeGraph( fileMapCache, packageJsonDeps, @@ -95,7 +96,10 @@ export async function buildProjectGraphUsingProjectFileMap( cachedFileData = fromCache.cachedFileData; } else { filesToProcess = fileMap; - cachedFileData = {}; + cachedFileData = { + nonProjectFiles: {}, + projectFileMap: {}, + }; } const context = createContext( @@ -155,7 +159,7 @@ async function buildProjectGraphUsingContext( nxJson: NxJsonConfiguration, knownExternalNodes: Record, ctx: CreateDependenciesContext, - cachedFileData: { [project: string]: { [file: string]: FileData } }, + cachedFileData: CachedFileData, projectGraphVersion: string ) { performance.mark('build project graph:start'); @@ -172,14 +176,20 @@ async function buildProjectGraphUsingContext( const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph); const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap); - for (const proj of Object.keys(cachedFileData)) { - for (const f of ctx.fileMap[proj] || []) { - const cached = cachedFileData[proj][f.file]; + for (const proj of Object.keys(cachedFileData.projectFileMap)) { + for (const f of ctx.fileMap.projectFileMap[proj] || []) { + const cached = cachedFileData.projectFileMap[proj][f.file]; if (cached && cached.deps) { f.deps = [...cached.deps]; } } } + for (const file of ctx.fileMap.nonProjectFiles) { + const cached = cachedFileData.nonProjectFiles[file.file]; + if (cached?.deps) { + file.deps = [...cached.deps]; + } + } applyImplicitDependencies(ctx.projects, updatedBuilder); diff --git a/packages/nx/src/project-graph/nx-deps-cache.ts b/packages/nx/src/project-graph/nx-deps-cache.ts index 2cdc29eb15ebb1..bf5d2ac7a078ad 100644 --- a/packages/nx/src/project-graph/nx-deps-cache.ts +++ b/packages/nx/src/project-graph/nx-deps-cache.ts @@ -231,7 +231,7 @@ export function shouldRecomputeWholeGraph( return false; } -type CachedFileData = { +export type CachedFileData = { nonProjectFiles: Record; projectFileMap: { [project: string]: Record }; }; @@ -291,11 +291,11 @@ function processNonProjectFiles( ) { const cachedHashMap = new Map(cachedFiles.map((f) => [f.file, f])); for (const f of nonProjectFiles) { - const cachedHash = cachedHashMap.get(f.file)?.hash; - if (!cachedHash || cachedHash !== f.hash) { + const cachedFile = cachedHashMap.get(f.file); + if (!cachedFile || cachedFile.hash !== f.hash) { filesToProcess.push(f); } else { - cachedFileData[f.file] = f; + cachedFileData[f.file] = cachedFile; } } } diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index a666a5c9b719df..a691f965f83f00 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -3,8 +3,11 @@ */ import { DependencyType, + FileData, + FileDataDependency, fileDataDepTarget, fileDataDepType, + FileMap, ProjectFileMap, ProjectGraph, ProjectGraphDependency, @@ -22,19 +25,42 @@ import { getFileMap } from './build-project-graph'; export class ProjectGraphBuilder { // TODO(FrozenPandaz): make this private readonly graph: ProjectGraph; + private readonly projectFileMap: ProjectFileMap; + private readonly nonProjectFiles: FileData[]; + private readonly allWorkspaceFiles: FileData[]; + readonly removedEdges: { [source: string]: Set } = {}; - constructor(graph?: ProjectGraph, fileMap?: ProjectFileMap) { + + constructor( + graph?: ProjectGraph, + projectFileMap?: ProjectFileMap, + nonProjectFiles?: FileMap['nonProjectFiles'] + ) { + if (!projectFileMap || !nonProjectFiles) { + const fileMap = getFileMap().fileMap; + projectFileMap ??= fileMap.projectFileMap; + nonProjectFiles ??= fileMap.nonProjectFiles; + } if (graph) { this.graph = graph; - this.projectFileMap = fileMap || getFileMap().fileMap?.projectFileMap; + this.projectFileMap = projectFileMap; + this.nonProjectFiles = nonProjectFiles; } else { this.graph = { nodes: {}, externalNodes: {}, dependencies: {}, }; - this.projectFileMap = fileMap || {}; + this.projectFileMap = projectFileMap || {}; + this.nonProjectFiles = nonProjectFiles || []; + } + + this.allWorkspaceFiles = this.nonProjectFiles; + for (const project in this.projectFileMap) { + this.allWorkspaceFiles = this.allWorkspaceFiles.concat( + this.projectFileMap[project] + ); } } @@ -234,6 +260,28 @@ export class ProjectGraphBuilder { } } } + for (const file of this.nonProjectFiles) { + if (file.deps) { + for (const dep of file.deps) { + if (!Array.isArray(dep)) { + throw new Error( + 'Cached data on non project files should be a tuple' + ); + } + const [source, target, type] = dep; + if (!source || !target || !type) { + throw new Error( + 'Cached dependencies for non project files should be a tuple of length 3.' + ); + } + this.graph.dependencies[source].push({ + source, + target, + type, + }); + } + } + } return this.graph; } @@ -257,8 +305,8 @@ export class ProjectGraphBuilder { { externalNodes: this.graph.externalNodes, fileMap: { - projectFileMap: { ...this.projectFileMap }, - nonProjectFiles: [], + projectFileMap: this.projectFileMap, + nonProjectFiles: this.nonProjectFiles, }, // the validators only really care about the keys on this. projects: this.graph.nodes as any, @@ -276,23 +324,28 @@ export class ProjectGraphBuilder { ); if (sourceFile) { - const fileData = getFileData( + let fileData = getProjectFileData( source, sourceFile, - this.graph.nodes, this.projectFileMap ); + const isProjectFileData = !!fileData; + fileData ??= getNonProjectFileData(sourceFile, this.allWorkspaceFiles); if (!fileData.deps) { fileData.deps = []; } + if ( !fileData.deps.find( (t) => fileDataDepTarget(t) === target && fileDataDepType(t) === type ) ) { - const dep: string | [string, string] = - type === 'static' ? target : [target, type]; + const dep: FileDataDependency = isProjectFileData + ? type === 'static' + ? target + : [target, type] + : [source, target, type]; fileData.deps.push(dep); } } else if (!isDuplicate) { @@ -491,8 +544,21 @@ function validateCommonDependencyRules( throw new Error(`External projects can't depend on internal projects`); } if ('sourceFile' in d && d.sourceFile) { - // Throws if source file is not a valid file within the source project. - getFileData(d.source, d.sourceFile, projects, fileMap.projectFileMap); + if (projects[d.source]) { + // Throws if source file is not a valid file within the source project. + // We can pass empty array for all workspace files here, since its not checked by the impl. + // We need all workspace files in here for the TODO comment though, so lets figure that out. + getFileData( + d.source, + d.sourceFile, + projects, + externalNodes, + fileMap.projectFileMap, + Object.values(fileMap.projectFileMap) + .flat() + .concat(fileMap.nonProjectFiles) + ); + } } } @@ -531,21 +597,45 @@ function validateStaticDependency( } } -function getFileData( +function getProjectFileData( source: string, sourceFile: string, - projects: Record, fileMap: ProjectFileMap ) { - const sourceProject = projects[source]; - if (!sourceProject) { - throw new Error(`Source project is not a project node: ${sourceProject}`); + let fileData = (fileMap[source] || []).find((f) => f.file === sourceFile); + if (fileData) { + return fileData; } - const fileData = (fileMap[source] || []).find((f) => f.file === sourceFile); +} + +function getNonProjectFileData( + sourceFile: string, + allWorkspaceFiles: FileData[] +) { + const fileData = allWorkspaceFiles.find((f) => f.file === sourceFile); if (!fileData) { - throw new Error( - `Source project ${source} does not have a file: ${sourceFile}` - ); + throw new Error(`Source file "${sourceFile}" is not in the workspace.`); } return fileData; } + +function getFileData( + source: string, + sourceFile: string, + projects: Record, + externalNodes: Record, + fileMap: ProjectFileMap, + allWorkspaceFiles: FileData[] +) { + const sourceProject = projects[source]; + const matchingExternalNode = externalNodes[source]; + + if (!sourceProject && !matchingExternalNode) { + throw new Error(`Source project is not a project node: ${sourceProject}`); + } + + return ( + getProjectFileData(source, sourceFile, fileMap) ?? + getNonProjectFileData(sourceFile, allWorkspaceFiles) + ); +} diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index e7ebbb8626f536..9ab713def47d8e 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -255,7 +255,7 @@ function buildAllWorkspaceFiles( globalFiles: FileData[] ): FileData[] { performance.mark('get-all-workspace-files:start'); - let fileData = Object.values(projectFileMap).flat(); + let fileData: FileData[] = Object.values(projectFileMap).flat(); fileData = fileData.concat(globalFiles); performance.mark('get-all-workspace-files:end'); diff --git a/packages/workspace/index.ts b/packages/workspace/index.ts index e506584ad01040..dd278a00fd4ebf 100644 --- a/packages/workspace/index.ts +++ b/packages/workspace/index.ts @@ -15,7 +15,6 @@ export { readWorkspaceConfig, readPackageJson, } from 'nx/src/project-graph/file-utils'; -export { ProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; export { getWorkspacePath,