From 69657477ff0b14888d50b400995afa4a02ca3c5a Mon Sep 17 00:00:00 2001 From: AgentEnder Date: Mon, 18 Sep 2023 14:14:27 -0400 Subject: [PATCH 1/2] feat(core): include non-project files on fileMap and filesToProcess --- .../devkit/CreateDependenciesContext.md | 8 +- docs/generated/devkit/ProjectGraphBuilder.md | 10 +- packages/nx/src/command-line/graph/graph.ts | 4 +- packages/nx/src/config/project-graph.ts | 5 + .../server/file-watching/changed-projects.ts | 4 +- .../nx/src/daemon/server/handle-hash-tasks.ts | 4 +- ...project-graph-incremental-recomputation.ts | 67 +++--- .../src/native/workspace/workspace_files.rs | 5 + .../plugins/js/lock-file/npm-parser.spec.ts | 100 +++++++-- .../plugins/js/lock-file/pnpm-parser.spec.ts | 90 ++++++-- .../plugins/js/lock-file/yarn-parser.spec.ts | 130 ++++++++--- .../js/package-json/create-package-json.ts | 4 +- ...explicit-package-json-dependencies.spec.ts | 12 +- .../explicit-package-json-dependencies.ts | 4 +- .../explicit-project-dependencies.spec.ts | 10 +- .../explicit-project-dependencies.ts | 4 +- .../src/project-graph/build-project-graph.ts | 106 +++++---- .../src/project-graph/file-map-utils.spec.ts | 70 +++--- .../nx/src/project-graph/file-map-utils.ts | 78 +++++-- .../src/project-graph/nx-deps-cache.spec.ts | 201 +++++++++++------- .../nx/src/project-graph/nx-deps-cache.ts | 83 ++++++-- .../project-graph/project-graph-builder.ts | 19 +- .../nx/src/project-graph/project-graph.ts | 17 +- .../implicit-project-dependencies.spec.ts | 10 +- .../utils/implicit-project-dependencies.ts | 8 +- .../utils/normalize-project-nodes.ts | 11 +- .../utils/retrieve-workspace-files.ts | 5 +- packages/nx/src/tasks-runner/run-command.ts | 6 +- packages/nx/src/utils/nx-plugin.ts | 5 +- 29 files changed, 713 insertions(+), 367 deletions(-) diff --git a/docs/generated/devkit/CreateDependenciesContext.md b/docs/generated/devkit/CreateDependenciesContext.md index 71595ae3bcffd..0bfdc1ac053a3 100644 --- a/docs/generated/devkit/CreateDependenciesContext.md +++ b/docs/generated/devkit/CreateDependenciesContext.md @@ -7,8 +7,8 @@ Context for [CreateDependencies](../../devkit/documents/CreateDependencies) ### Properties - [externalNodes](../../devkit/documents/CreateDependenciesContext#externalnodes): Record<string, ProjectGraphExternalNode> -- [fileMap](../../devkit/documents/CreateDependenciesContext#filemap): ProjectFileMap -- [filesToProcess](../../devkit/documents/CreateDependenciesContext#filestoprocess): ProjectFileMap +- [fileMap](../../devkit/documents/CreateDependenciesContext#filemap): FileMap +- [filesToProcess](../../devkit/documents/CreateDependenciesContext#filestoprocess): FileMap - [nxJsonConfiguration](../../devkit/documents/CreateDependenciesContext#nxjsonconfiguration): NxJsonConfiguration<string[] | "\*"> - [projects](../../devkit/documents/CreateDependenciesContext#projects): Record<string, ProjectConfiguration> - [workspaceRoot](../../devkit/documents/CreateDependenciesContext#workspaceroot): string @@ -25,7 +25,7 @@ The external nodes that have been added to the graph. ### fileMap -• `Readonly` **fileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) +• `Readonly` **fileMap**: `FileMap` All files in the workspace @@ -33,7 +33,7 @@ All files in the workspace ### filesToProcess -• `Readonly` **filesToProcess**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) +• `Readonly` **filesToProcess**: `FileMap` Files changes since last invocation diff --git a/docs/generated/devkit/ProjectGraphBuilder.md b/docs/generated/devkit/ProjectGraphBuilder.md index efd6c1190ad2f..7e0669d961a8f 100644 --- a/docs/generated/devkit/ProjectGraphBuilder.md +++ b/docs/generated/devkit/ProjectGraphBuilder.md @@ -14,8 +14,8 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ### Properties -- [fileMap](../../devkit/documents/ProjectGraphBuilder#filemap): ProjectFileMap - [graph](../../devkit/documents/ProjectGraphBuilder#graph): ProjectGraph +- [projectFileMap](../../devkit/documents/ProjectGraphBuilder#projectfilemap): ProjectFileMap - [removedEdges](../../devkit/documents/ProjectGraphBuilder#removededges): Object ### Methods @@ -51,15 +51,15 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ## Properties -### fileMap +### graph -• `Private` `Readonly` **fileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) +• `Readonly` **graph**: [`ProjectGraph`](../../devkit/documents/ProjectGraph) --- -### graph +### projectFileMap -• `Readonly` **graph**: [`ProjectGraph`](../../devkit/documents/ProjectGraph) +• `Private` `Readonly` **projectFileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) --- diff --git a/packages/nx/src/command-line/graph/graph.ts b/packages/nx/src/command-line/graph/graph.ts index 0da8eae3e01d4..8a64150f35cbd 100644 --- a/packages/nx/src/command-line/graph/graph.ts +++ b/packages/nx/src/command-line/graph/graph.ts @@ -25,7 +25,7 @@ import { import { TaskGraph } from '../../config/task-graph'; import { daemonClient } from '../../daemon/client/client'; import { Server } from 'net'; -import { readProjectFileMapCache } from '../../project-graph/nx-deps-cache'; +import { readFileMapCache } from '../../project-graph/nx-deps-cache'; import { getAffectedGraphNodes } from '../affected/affected'; import { splitArgsIntoNxArgsAndOverrides } from '../../utils/command-line-utils'; @@ -576,7 +576,7 @@ async function createDepGraphClientResponse( let graph = pruneExternalNodes( await createProjectGraphAsync({ exitOnError: true }) ); - let fileMap = readProjectFileMapCache().projectFileMap; + let fileMap = readFileMapCache().fileMap.projectFileMap; performance.mark('project graph watch calculation:end'); performance.mark('project graph response generation:start'); diff --git a/packages/nx/src/config/project-graph.ts b/packages/nx/src/config/project-graph.ts index 872f3970c183a..14b90a7e1f65b 100644 --- a/packages/nx/src/config/project-graph.ts +++ b/packages/nx/src/config/project-graph.ts @@ -22,6 +22,11 @@ export function fileDataDepType(dep: string | [string, string]) { return typeof dep === 'string' ? 'static' : dep[1]; } +export interface FileMap { + nonProjectFiles: FileData[]; + projectFileMap: ProjectFileMap; +} + /** * A list of files separated by the project they belong to */ diff --git a/packages/nx/src/daemon/server/file-watching/changed-projects.ts b/packages/nx/src/daemon/server/file-watching/changed-projects.ts index 486161d8aaa5b..091bf9570c688 100644 --- a/packages/nx/src/daemon/server/file-watching/changed-projects.ts +++ b/packages/nx/src/daemon/server/file-watching/changed-projects.ts @@ -1,5 +1,5 @@ import { performance } from 'perf_hooks'; -import { projectFileMapWithFiles } from '../project-graph-incremental-recomputation'; +import { fileMapWithFiles } from '../project-graph-incremental-recomputation'; export type ChangedFile = { path: string; @@ -38,7 +38,7 @@ export function getProjectsAndGlobalChanges( const fileToProjectMap: Record = {}; for (const [projectName, projectFiles] of Object.entries( - projectFileMapWithFiles?.projectFileMap ?? {} + fileMapWithFiles?.fileMap?.projectFileMap ?? {} )) { for (const projectFile of projectFiles) { fileToProjectMap[projectFile.file] = projectName; diff --git a/packages/nx/src/daemon/server/handle-hash-tasks.ts b/packages/nx/src/daemon/server/handle-hash-tasks.ts index f85c931f14661..fca80b0c822e9 100644 --- a/packages/nx/src/daemon/server/handle-hash-tasks.ts +++ b/packages/nx/src/daemon/server/handle-hash-tasks.ts @@ -19,14 +19,14 @@ export async function handleHashTasks(payload: { }) { setHashEnv(payload.env); - const { projectGraph, allWorkspaceFiles, projectFileMap } = + const { projectGraph, allWorkspaceFiles, fileMap } = await getCachedSerializedProjectGraphPromise(); const nxJson = readNxJson(); if (projectGraph !== storedProjectGraph) { storedProjectGraph = projectGraph; storedHasher = new InProcessTaskHasher( - projectFileMap, + fileMap?.projectFileMap, allWorkspaceFiles, projectGraph, nxJson, diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index f65f755cfd88c..fb6bf04b75a16 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -1,16 +1,17 @@ import { performance } from 'perf_hooks'; import { FileData, + FileMap, ProjectFileMap, ProjectGraph, ProjectGraphExternalNode, } from '../../config/project-graph'; -import { buildProjectGraphUsingProjectFileMap } from '../../project-graph/build-project-graph'; -import { updateProjectFileMap } from '../../project-graph/file-map-utils'; +import { buildProjectGraphUsingProjectFileMap as buildProjectGraphUsingFileMap } from '../../project-graph/build-project-graph'; +import { updateFileMap } from '../../project-graph/file-map-utils'; import { nxProjectGraph, - ProjectFileMapCache, - readProjectFileMapCache, + FileMapCache, + readFileMapCache, } from '../../project-graph/nx-deps-cache'; import { fileExists } from '../../utils/fileutils'; import { notifyFileWatcherSockets } from './file-watching/file-watcher-sockets'; @@ -35,14 +36,14 @@ import { let cachedSerializedProjectGraphPromise: Promise<{ error: Error | null; projectGraph: ProjectGraph | null; - projectFileMap: ProjectFileMap | null; + fileMap: FileMap | null; allWorkspaceFiles: FileData[] | null; serializedProjectGraph: string | null; }>; -export let projectFileMapWithFiles: - | { projectFileMap: ProjectFileMap; allWorkspaceFiles: FileData[] } +export let fileMapWithFiles: + | { fileMap: FileMap; allWorkspaceFiles: FileData[] } | undefined; -export let currentProjectFileMapCache: ProjectFileMapCache | undefined; +export let currentProjectFileMapCache: FileMapCache | undefined; export let currentProjectGraph: ProjectGraph | undefined; const collectedUpdatedFiles = new Set(); @@ -78,7 +79,7 @@ export async function getCachedSerializedProjectGraphPromise() { error: e, serializedProjectGraph: null, projectGraph: null, - projectFileMap: null, + fileMap: null, allWorkspaceFiles: null, }; } @@ -197,22 +198,19 @@ async function processCollectedUpdatedAndDeletedFiles() { if (workspaceConfigHash !== storedWorkspaceConfigHash) { storedWorkspaceConfigHash = workspaceConfigHash; - ({ externalNodes: knownExternalNodes, ...projectFileMapWithFiles } = + ({ externalNodes: knownExternalNodes, ...fileMapWithFiles } = await retrieveWorkspaceFiles(workspaceRoot, nxJson)); } else { - if (projectFileMapWithFiles) { - projectFileMapWithFiles = updateProjectFileMap( + if (fileMapWithFiles) { + fileMapWithFiles = updateFileMap( projectNodes, - projectFileMapWithFiles.projectFileMap, - projectFileMapWithFiles.allWorkspaceFiles, + fileMapWithFiles.fileMap, + fileMapWithFiles.allWorkspaceFiles, new Map(Object.entries(updatedFileHashes)), deletedFiles ); } else { - projectFileMapWithFiles = await retrieveWorkspaceFiles( - workspaceRoot, - nxJson - ); + fileMapWithFiles = await retrieveWorkspaceFiles(workspaceRoot, nxJson); } } @@ -240,7 +238,7 @@ async function processFilesAndCreateAndSerializeProjectGraph() { return Promise.resolve({ error: err, projectGraph: null, - projectFileMap: null, + fileMap: null, allWorkspaceFiles: null, serializedProjectGraph: null, }); @@ -253,10 +251,13 @@ function copyFileData(d: FileData[]) { return d.map((t) => ({ ...t })); } -function copyFileMap(m: ProjectFileMap) { - const c = {}; - for (let p of Object.keys(m)) { - c[p] = copyFileData(m[p]); +function copyFileMap(m: FileMap) { + const c: FileMap = { + nonProjectFiles: copyFileData(m.nonProjectFiles), + projectFileMap: {}, + }; + for (let p of Object.keys(m.projectFileMap)) { + c[p] = copyFileData(m.projectFileMap[p]); } return c; } @@ -264,7 +265,7 @@ function copyFileMap(m: ProjectFileMap) { async function createAndSerializeProjectGraph(): Promise<{ error: string | null; projectGraph: ProjectGraph | null; - projectFileMap: ProjectFileMap | null; + fileMap: FileMap | null; allWorkspaceFiles: FileData[] | null; serializedProjectGraph: string | null; }> { @@ -274,17 +275,15 @@ async function createAndSerializeProjectGraph(): Promise<{ workspaceRoot, readNxJson(workspaceRoot) ); - const projectFileMap = copyFileMap(projectFileMapWithFiles.projectFileMap); - const allWorkspaceFiles = copyFileData( - projectFileMapWithFiles.allWorkspaceFiles - ); + const fileMap = copyFileMap(fileMapWithFiles.fileMap); + const allWorkspaceFiles = copyFileData(fileMapWithFiles.allWorkspaceFiles); const { projectGraph, projectFileMapCache } = - await buildProjectGraphUsingProjectFileMap( + await buildProjectGraphUsingFileMap( projectConfigurations.projectNodes, knownExternalNodes, - projectFileMap, + fileMap, allWorkspaceFiles, - currentProjectFileMapCache || readProjectFileMapCache(), + currentProjectFileMapCache || readFileMapCache(), true ); currentProjectFileMapCache = projectFileMapCache; @@ -309,7 +308,7 @@ async function createAndSerializeProjectGraph(): Promise<{ return { error: null, projectGraph, - projectFileMap, + fileMap, allWorkspaceFiles, serializedProjectGraph, }; @@ -320,7 +319,7 @@ async function createAndSerializeProjectGraph(): Promise<{ return { error: e, projectGraph: null, - projectFileMap: null, + fileMap: null, allWorkspaceFiles: null, serializedProjectGraph: null, }; @@ -329,7 +328,7 @@ async function createAndSerializeProjectGraph(): Promise<{ async function resetInternalState() { cachedSerializedProjectGraphPromise = undefined; - projectFileMapWithFiles = undefined; + fileMapWithFiles = undefined; currentProjectFileMapCache = undefined; currentProjectGraph = undefined; collectedUpdatedFiles.clear(); diff --git a/packages/nx/src/native/workspace/workspace_files.rs b/packages/nx/src/native/workspace/workspace_files.rs index 891b1fe1302c1..eeae027316227 100644 --- a/packages/nx/src/native/workspace/workspace_files.rs +++ b/packages/nx/src/native/workspace/workspace_files.rs @@ -20,6 +20,11 @@ pub struct NxWorkspaceFiles { pub external_nodes: HashMap, } +pub struct FileMap { + pub project_file_map: HashMap>, + pub non_project_files: Vec +} + pub(super) fn get_files( globs: Vec, parse_configurations: ConfigurationParser, diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts index 240e6e6396408..2ebb0dbb3a8b7 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts @@ -40,8 +40,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -96,8 +102,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -175,8 +187,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -265,8 +283,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -395,8 +419,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -514,8 +544,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -557,8 +593,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -604,8 +646,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -662,8 +710,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -722,8 +776,14 @@ describe('NPM lock file utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts index ffbe1692891e7..bdbcea297d715 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts @@ -151,8 +151,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -212,8 +218,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -259,8 +271,14 @@ describe('pnpm LockFile utility', () => { const appCtx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -337,8 +355,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -448,8 +472,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -516,8 +546,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -571,8 +607,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -639,8 +681,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -718,8 +766,14 @@ describe('pnpm LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts index 210a94f93c072..41e83ac85c542 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts @@ -192,8 +192,14 @@ describe('yarn LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -510,8 +516,14 @@ describe('yarn LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -577,8 +589,14 @@ describe('yarn LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -716,8 +734,14 @@ describe('yarn LockFile utility', () => { const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -793,8 +817,14 @@ __metadata: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1267,8 +1297,14 @@ nx-cloud@latest: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1476,8 +1512,14 @@ nx-cloud@latest: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1533,8 +1575,14 @@ nx-cloud@latest: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1676,8 +1724,14 @@ type-fest@^0.20.2: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1789,8 +1843,14 @@ __metadata: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -1912,8 +1972,14 @@ __metadata: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -2149,8 +2215,14 @@ __metadata: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; @@ -2536,8 +2608,14 @@ __metadata: const ctx: CreateDependenciesContext = { projects: {}, externalNodes, - fileMap: {}, - filesToProcess: {}, + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + filesToProcess: { + nonProjectFiles: [], + projectFileMap: {}, + }, nxJsonConfiguration: null, workspaceRoot: '/virtual', }; diff --git a/packages/nx/src/plugins/js/package-json/create-package-json.ts b/packages/nx/src/plugins/js/package-json/create-package-json.ts index e67be3838eb79..829f4ab2d5955 100644 --- a/packages/nx/src/plugins/js/package-json/create-package-json.ts +++ b/packages/nx/src/plugins/js/package-json/create-package-json.ts @@ -14,7 +14,7 @@ import { getTargetInputs, } from '../../../hasher/task-hasher'; import { readNxJson } from '../../../config/configuration'; -import { readProjectFileMapCache } from '../../../project-graph/nx-deps-cache'; +import { readFileMapCache } from '../../../project-graph/nx-deps-cache'; import { join } from 'path'; interface NpmDeps { @@ -193,7 +193,7 @@ export function findProjectsNpmDependencies( fileMap?: ProjectFileMap ): NpmDeps { if (fileMap == null) { - fileMap = readProjectFileMapCache()?.projectFileMap || {}; + fileMap = readFileMapCache()?.fileMap?.projectFileMap || {}; } const { selfInputs, dependencyInputs } = target diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts index 2b063652095d7..a565b7db1b9c5 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts @@ -5,7 +5,7 @@ import { buildExplicitPackageJsonDependencies } from './explicit-package-json-de import { ProjectGraphProjectNode } from '../../../../config/project-graph'; import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; -import { createProjectFileMap } from '../../../../project-graph/file-map-utils'; +import { createFileMap } from '../../../../project-graph/file-map-utils'; import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; import { getAllFileDataInContext } from '../../../../utils/workspace-context'; @@ -71,12 +71,12 @@ describe('explicit package json dependencies', () => { }, }; - const projectFileMap = createProjectFileMap( + const fileMap = createFileMap( projectsConfigurations as any, getAllFileDataInContext(tempFs.tempDir) - ).projectFileMap; + ).fileMap; - const builder = new ProjectGraphBuilder(undefined, projectFileMap); + const builder = new ProjectGraphBuilder(undefined, fileMap.projectFileMap); Object.values(projects).forEach((p) => { builder.addNode(p); }); @@ -90,11 +90,11 @@ describe('explicit package json dependencies', () => { }); ctx = { - fileMap: projectFileMap, + fileMap: fileMap, externalNodes: builder.getUpdatedProjectGraph().externalNodes, projects: projectsConfigurations.projects, nxJsonConfiguration, - filesToProcess: projectFileMap, + filesToProcess: fileMap, workspaceRoot: tempFs.tempDir, }; }); diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts index 8f2bc0c95fd08..cb5f9d3eac083 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts @@ -21,8 +21,8 @@ export function buildExplicitPackageJsonDependencies( const res: RawProjectGraphDependency[] = []; let packageNameMap = undefined; const nodes = Object.values(ctx.projects); - Object.keys(ctx.filesToProcess).forEach((source) => { - Object.values(ctx.filesToProcess[source]).forEach((f) => { + Object.keys(ctx.filesToProcess.projectFileMap).forEach((source) => { + Object.values(ctx.filesToProcess.projectFileMap[source]).forEach((f) => { if (isPackageJsonAtProjectRoot(nodes, f.file)) { // we only create the package name map once and only if a package.json file changes packageNameMap = diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts index ab0f3522a9834..65f180f588a8c 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts @@ -562,15 +562,17 @@ async function createContext( setupWorkspaceContext(tempFs.tempDir); - const { projectFileMap, projectConfigurations } = - await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + const { fileMap, projectConfigurations } = await retrieveWorkspaceFiles( + tempFs.tempDir, + nxJson + ); return { externalNodes: builder.getUpdatedProjectGraph().externalNodes, projects: projectConfigurations.projects, nxJsonConfiguration: nxJson, - filesToProcess: projectFileMap, - fileMap: projectFileMap, + filesToProcess: fileMap, + fileMap: fileMap, workspaceRoot: tempFs.tempDir, }; } diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts index 2244ddd3e0d67..948e917fd09b0 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts @@ -79,7 +79,9 @@ export function buildExplicitTypeScriptDependencies( moduleExtensions.push('.vue'); } - for (const [project, fileData] of Object.entries(ctx.fileMap)) { + for (const [project, fileData] of Object.entries( + ctx.fileMap.projectFileMap + )) { filesToProcess[project] ??= []; for (const { file } of fileData) { if (moduleExtensions.some((ext) => file.endsWith(ext))) { diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 16c151bbb9033..36fc2a3fb2256 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -6,15 +6,22 @@ import { FileData } from './file-utils'; import { createProjectFileMapCache, extractCachedFileData, - ProjectFileMapCache, + FileMapCache, shouldRecomputeWholeGraph, writeCache, } from './nx-deps-cache'; import { applyImplicitDependencies } from './utils/implicit-project-dependencies'; import { normalizeProjectNodes } from './utils/normalize-project-nodes'; -import { isNxPluginV1, isNxPluginV2, loadNxPlugins } from '../utils/nx-plugin'; +import { + CreateDependenciesContext, + CreateNodesContext, + isNxPluginV1, + isNxPluginV2, + loadNxPlugins, +} from '../utils/nx-plugin'; import { getRootTsConfigPath } from '../plugins/js/utils/typescript'; import { + FileMap, ProjectFileMap, ProjectGraph, ProjectGraphExternalNode, @@ -28,35 +35,41 @@ import { readNxJson } from '../config/configuration'; import { existsSync } from 'fs'; import { PackageJson } from '../utils/package-json'; -let storedProjectFileMap: ProjectFileMap | null = null; +let storedFileMap: FileMap | null = null; let storedAllWorkspaceFiles: FileData[] | null = null; -export function getProjectFileMap(): { - projectFileMap: ProjectFileMap; +export function getFileMap(): { + fileMap: FileMap; allWorkspaceFiles: FileData[]; } { - if (!!storedProjectFileMap) { + if (!!storedFileMap) { return { - projectFileMap: storedProjectFileMap, + fileMap: storedFileMap, allWorkspaceFiles: storedAllWorkspaceFiles, }; } else { - return { projectFileMap: {}, allWorkspaceFiles: [] }; + return { + fileMap: { + nonProjectFiles: [], + projectFileMap: {}, + }, + allWorkspaceFiles: [], + }; } } export async function buildProjectGraphUsingProjectFileMap( projects: Record, externalNodes: Record, - projectFileMap: ProjectFileMap, + fileMap: FileMap, allWorkspaceFiles: FileData[], - fileMap: ProjectFileMapCache | null, + fileMapCache: FileMapCache | null, shouldWriteCache: boolean ): Promise<{ projectGraph: ProjectGraph; - projectFileMapCache: ProjectFileMapCache; + projectFileMapCache: FileMapCache; }> { - storedProjectFileMap = projectFileMap; + storedFileMap = fileMap; storedAllWorkspaceFiles = allWorkspaceFiles; const nxJson = readNxJson(); @@ -70,25 +83,26 @@ export async function buildProjectGraphUsingProjectFileMap( const useCacheData = fileMap && !shouldRecomputeWholeGraph( - fileMap, + fileMapCache, packageJsonDeps, projects, nxJson, rootTsConfig ); if (useCacheData) { - const fromCache = extractCachedFileData(projectFileMap, fileMap); + const fromCache = extractCachedFileData(fileMap, fileMapCache); filesToProcess = fromCache.filesToProcess; cachedFileData = fromCache.cachedFileData; } else { - filesToProcess = projectFileMap; + filesToProcess = fileMap; cachedFileData = {}; } const context = createContext( projects, nxJson, - projectFileMap, + externalNodes, + fileMap, filesToProcess ); let projectGraph = await buildProjectGraphUsingContext( @@ -101,7 +115,7 @@ export async function buildProjectGraphUsingProjectFileMap( const projectFileMapCache = createProjectFileMapCache( nxJson, packageJsonDeps, - projectFileMap, + fileMap, rootTsConfig ); if (shouldWriteCache) { @@ -140,13 +154,13 @@ function readCombinedDeps() { async function buildProjectGraphUsingContext( nxJson: NxJsonConfiguration, knownExternalNodes: Record, - ctx: ProjectGraphProcessorContext, + ctx: CreateDependenciesContext, cachedFileData: { [project: string]: { [file: string]: FileData } }, projectGraphVersion: string ) { performance.mark('build project graph:start'); - const builder = new ProjectGraphBuilder(null, ctx.fileMap); + const builder = new ProjectGraphBuilder(null, ctx.fileMap.projectFileMap); builder.setVersion(projectGraphVersion); for (const node in knownExternalNodes) { builder.addExternalNode(knownExternalNodes[node]); @@ -157,7 +171,7 @@ async function buildProjectGraphUsingContext( const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph); - const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap); + const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap); for (const proj of Object.keys(cachedFileData)) { for (const f of ctx.fileMap[proj] || []) { const cached = cachedFileData[proj][f.file]; @@ -167,7 +181,7 @@ async function buildProjectGraphUsingContext( } } - applyImplicitDependencies(ctx.projectsConfigurations, updatedBuilder); + applyImplicitDependencies(ctx.projects, updatedBuilder); const finalGraph = updatedBuilder.getUpdatedProjectGraph(); @@ -184,9 +198,10 @@ async function buildProjectGraphUsingContext( function createContext( projects: Record, nxJson: NxJsonConfiguration, - fileMap: ProjectFileMap, - filesToProcess: ProjectFileMap -): ProjectGraphProcessorContext { + externalNodes: Record, + fileMap: FileMap, + filesToProcess: FileMap +): CreateDependenciesContext { const clonedProjects = Object.keys(projects).reduce((map, projectName) => { map[projectName] = { ...projects[projectName], @@ -195,22 +210,16 @@ function createContext( }, {} as Record); return { nxJsonConfiguration: nxJson, - workspace: { - version: 2, - projects: clonedProjects, - ...nxJson, - }, - projectsConfigurations: { - version: 2, - projects: clonedProjects, - }, + projects: clonedProjects, + externalNodes, + workspaceRoot, fileMap, filesToProcess, }; } async function updateProjectGraphWithPlugins( - context: ProjectGraphProcessorContext, + context: CreateDependenciesContext, initProjectGraph: ProjectGraph ) { const plugins = await loadNxPlugins(context.nxJsonConfiguration?.plugins); @@ -229,7 +238,20 @@ async function updateProjectGraphWithPlugins( // 'Nx has recently released a v2 model for project graph plugins. The `processProjectGraph` method is deprecated. Plugins should use some combination of `createNodes` and `createDependencies` instead.', // ], // }); - graph = await plugin.processProjectGraph(graph, context); + graph = await plugin.processProjectGraph(graph, { + ...context, + projectsConfigurations: { + projects: context.projects, + version: 2, + }, + fileMap: context.fileMap.projectFileMap, + filesToProcess: context.filesToProcess.projectFileMap, + workspace: { + version: 2, + projects: context.projects, + ...context.nxJsonConfiguration, + }, + }); } } catch (e) { let message = `Failed to process the project graph with "${plugin.name}".`; @@ -243,15 +265,11 @@ async function updateProjectGraphWithPlugins( for (const plugin of plugins) { try { if (isNxPluginV2(plugin) && plugin.createDependencies) { - const builder = new ProjectGraphBuilder(graph, context.fileMap); - const newDependencies = await plugin.createDependencies({ - externalNodes: graph.externalNodes, - fileMap: context.fileMap, - filesToProcess: context.filesToProcess, - nxJsonConfiguration: context.nxJsonConfiguration, - projects: context.projectsConfigurations.projects, - workspaceRoot: workspaceRoot, - }); + const builder = new ProjectGraphBuilder( + graph, + context.fileMap.projectFileMap + ); + const newDependencies = await plugin.createDependencies(context); for (const targetProjectDependency of newDependencies) { builder.addDependency( targetProjectDependency.source, diff --git a/packages/nx/src/project-graph/file-map-utils.spec.ts b/packages/nx/src/project-graph/file-map-utils.spec.ts index 1a9eb230f325c..e7287e4efeb2e 100644 --- a/packages/nx/src/project-graph/file-map-utils.spec.ts +++ b/packages/nx/src/project-graph/file-map-utils.spec.ts @@ -1,5 +1,5 @@ import { ProjectType } from '../config/workspace-json-project-json'; -import { createProjectFileMap, updateProjectFileMap } from './file-map-utils'; +import { createFileMap, updateFileMap } from './file-map-utils'; describe('fileMapUtils', () => { describe('createFileMap', () => { @@ -31,15 +31,18 @@ describe('fileMapUtils', () => { { file: 'tools/myfile.txt', hash: 'some-hash' }, ]; - const result = createProjectFileMap(projectsConfigurations, files); + const result = createFileMap(projectsConfigurations, files); expect(result).toEqual({ - projectFileMap: { - demo: [{ file: 'apps/demo/src/main.ts', hash: 'some-hash' }], - 'demo-e2e': [ - { file: 'apps/demo-e2e/src/main.ts', hash: 'some-hash' }, - ], - ui: [{ file: 'libs/ui/src/index.ts', hash: 'some-hash' }], + fileMap: { + projectFileMap: { + demo: [{ file: 'apps/demo/src/main.ts', hash: 'some-hash' }], + 'demo-e2e': [ + { file: 'apps/demo-e2e/src/main.ts', hash: 'some-hash' }, + ], + ui: [{ file: 'libs/ui/src/index.ts', hash: 'some-hash' }], + }, + nonProjectFiles: [{ file: 'tools/myfile.txt', hash: 'some-hash' }], }, allWorkspaceFiles: [ { file: 'apps/demo/src/main.ts', hash: 'some-hash' }, @@ -76,6 +79,7 @@ describe('fileMapUtils', () => { { file: 'libs/ui/src/index.ts', hash: 'some-hash' }, { file: 'libs/ui/src/second.ts', hash: 'some-hash' }, { file: 'tools/myfile.txt', hash: 'some-hash' }, + { file: 'tools/secondfile.txt', hash: 'some-hash' }, ]; const projectFileMap = { @@ -86,33 +90,47 @@ describe('fileMapUtils', () => { { file: 'libs/ui/src/second.ts', hash: 'some-hash' }, ], }; - const result = updateProjectFileMap( - projectsConfigurations, + + const fileMap = { projectFileMap, + allWorkspaceFiles: files, + nonProjectFiles: files.filter( + (f) => + !Object.values(projectFileMap).some((arr) => + arr.some((projectFile) => projectFile.file === f.file) + ) + ), + }; + const result = updateFileMap( + projectsConfigurations, + fileMap, files, new Map([ ['apps/demo/src/main.ts', 'demo-main-update'], ['apps/demo/src/new-main.ts', 'new-main-hash'], ]), - ['libs/ui/src/second.ts'] + ['libs/ui/src/second.ts', 'tools/secondfile.txt'] ); expect(result).toEqual({ - projectFileMap: { - demo: [ - { - file: 'apps/demo/src/main.ts', - hash: 'demo-main-update', - }, - { - file: 'apps/demo/src/new-main.ts', - hash: 'new-main-hash', - }, - ], - 'demo-e2e': [ - { file: 'apps/demo-e2e/src/main.ts', hash: 'some-hash' }, - ], - ui: [{ file: 'libs/ui/src/index.ts', hash: 'some-hash' }], + fileMap: { + projectFileMap: { + demo: [ + { + file: 'apps/demo/src/main.ts', + hash: 'demo-main-update', + }, + { + file: 'apps/demo/src/new-main.ts', + hash: 'new-main-hash', + }, + ], + 'demo-e2e': [ + { file: 'apps/demo-e2e/src/main.ts', hash: 'some-hash' }, + ], + ui: [{ file: 'libs/ui/src/index.ts', hash: 'some-hash' }], + }, + nonProjectFiles: [{ file: 'tools/myfile.txt', hash: 'some-hash' }], }, allWorkspaceFiles: [ { file: 'apps/demo/src/main.ts', hash: 'demo-main-update' }, diff --git a/packages/nx/src/project-graph/file-map-utils.ts b/packages/nx/src/project-graph/file-map-utils.ts index 9ed9e2c575f5e..5d966eb5d2021 100644 --- a/packages/nx/src/project-graph/file-map-utils.ts +++ b/packages/nx/src/project-graph/file-map-utils.ts @@ -1,5 +1,6 @@ import { FileData, + FileMap, ProjectFileMap, ProjectGraph, } from '../config/project-graph'; @@ -28,55 +29,79 @@ export async function createProjectFileMapUsingProjectGraph( files = getAllFileDataInContext(workspaceRoot); } - return createProjectFileMap(configs, files).projectFileMap; + return createFileMap(configs, files).fileMap.projectFileMap; } -export function createProjectFileMap( +export function createFileMap( projectsConfigurations: ProjectsConfigurations, allWorkspaceFiles: FileData[] -): { projectFileMap: ProjectFileMap; allWorkspaceFiles: FileData[] } { +): { + allWorkspaceFiles: FileData[]; + fileMap: FileMap; +} { const projectFileMap: ProjectFileMap = {}; const projectRootMappings = createProjectRootMappingsFromProjectConfigurations( projectsConfigurations.projects ); + const nonProjectFiles: FileData[] = []; for (const projectName of Object.keys(projectsConfigurations.projects)) { projectFileMap[projectName] ??= []; } for (const f of allWorkspaceFiles) { const projectFileMapKey = findProjectForPath(f.file, projectRootMappings); - const matchingProjectFiles = projectFileMap[projectFileMapKey]; - if (matchingProjectFiles) { - matchingProjectFiles.push(f); + if (projectFileMapKey) { + const matchingProjectFiles = projectFileMap[projectFileMapKey]; + if (matchingProjectFiles) { + matchingProjectFiles.push(f); + } + } else { + nonProjectFiles.push(f); } } - return { projectFileMap, allWorkspaceFiles }; + return { + allWorkspaceFiles, + fileMap: { + projectFileMap, + nonProjectFiles, + }, + }; } -export function updateProjectFileMap( +export function updateFileMap( projectsConfigurations: Record, - projectFileMap: ProjectFileMap, + { projectFileMap, nonProjectFiles }: FileMap, allWorkspaceFiles: FileData[], updatedFiles: Map, deletedFiles: string[] -): { projectFileMap: ProjectFileMap; allWorkspaceFiles: FileData[] } { +): { fileMap: FileMap; allWorkspaceFiles: FileData[] } { const projectRootMappings = createProjectRootMappingsFromProjectConfigurations(projectsConfigurations); + let nonProjectFilesMap = new Map(nonProjectFiles.map((f) => [f.file, f])); for (const f of updatedFiles.keys()) { - const matchingProjectFiles = - projectFileMap[findProjectForPath(f, projectRootMappings)] ?? []; - if (matchingProjectFiles) { - const fileData: FileData = matchingProjectFiles.find((t) => t.file === f); - if (fileData) { - fileData.hash = updatedFiles.get(f); - } else { - matchingProjectFiles.push({ - file: f, - hash: updatedFiles.get(f), - }); + const project = findProjectForPath(f, projectRootMappings); + if (project) { + const matchingProjectFiles = projectFileMap[project] ?? []; + if (matchingProjectFiles) { + const fileData: FileData = matchingProjectFiles.find( + (t) => t.file === f + ); + if (fileData) { + fileData.hash = updatedFiles.get(f); + } else { + matchingProjectFiles.push({ + file: f, + hash: updatedFiles.get(f), + }); + } } + } else { + const hash = updatedFiles.get(f); + const entry = nonProjectFilesMap.get(f) ?? { file: f, hash }; + entry.hash = hash; + nonProjectFilesMap.set(f, entry); } const fileData: FileData = allWorkspaceFiles.find((t) => t.file === f); @@ -99,10 +124,19 @@ export function updateProjectFileMap( matchingProjectFiles.splice(index, 1); } } + if (nonProjectFilesMap.has(f)) { + nonProjectFilesMap.delete(f); + } const index = allWorkspaceFiles.findIndex((t) => t.file === f); if (index > -1) { allWorkspaceFiles.splice(index, 1); } } - return { projectFileMap, allWorkspaceFiles }; + return { + fileMap: { + projectFileMap, + nonProjectFiles: Array.from(nonProjectFilesMap.values()), + }, + allWorkspaceFiles, + }; } diff --git a/packages/nx/src/project-graph/nx-deps-cache.spec.ts b/packages/nx/src/project-graph/nx-deps-cache.spec.ts index 1935f32e2f947..8bfd3246c7516 100644 --- a/packages/nx/src/project-graph/nx-deps-cache.spec.ts +++ b/packages/nx/src/project-graph/nx-deps-cache.spec.ts @@ -1,7 +1,7 @@ import { createProjectFileMapCache as _createCache, extractCachedFileData, - ProjectFileMapCache, + FileMapCache, shouldRecomputeWholeGraph, } from './nx-deps-cache'; import { ProjectConfiguration } from '../config/workspace-json-project-json'; @@ -52,9 +52,12 @@ describe('nx deps utils', () => { expect( shouldRecomputeWholeGraph( createCache({ - projectFileMap: { - 'renamed-mylib': [], - } as any, + fileMap: { + projectFileMap: { + 'renamed-mylib': [], + } as any, + nonProjectFiles: [], + }, }), createPackageJsonDeps({}), createProjectsConfiguration({}), @@ -119,14 +122,7 @@ describe('nx deps utils', () => { it('should return the cache project graph when nothing has changed', () => { const r = extractCachedFileData( { - mylib: [ - { - file: 'index.ts', - hash: 'hash1', - }, - ], - }, - createCache({ + nonProjectFiles: [], projectFileMap: { mylib: [ { @@ -135,14 +131,33 @@ describe('nx deps utils', () => { }, ], }, + }, + createCache({ + fileMap: { + nonProjectFiles: [], + projectFileMap: { + mylib: [ + { + file: 'index.ts', + hash: 'hash1', + }, + ], + }, + }, }) ); - expect(r.filesToProcess).toEqual({}); + expect(r.filesToProcess).toEqual({ + projectFileMap: {}, + nonProjectFiles: [], + }); expect(r.cachedFileData).toEqual({ - mylib: { - 'index.ts': { - file: 'index.ts', - hash: 'hash1', + nonProjectFiles: {}, + projectFileMap: { + mylib: { + 'index.ts': { + file: 'index.ts', + hash: 'hash1', + }, }, }, }); @@ -151,20 +166,7 @@ describe('nx deps utils', () => { it('should handle cases when new projects are added', () => { const r = extractCachedFileData( { - mylib: [ - { - file: 'index.ts', - hash: 'hash1', - }, - ], - secondlib: [ - { - file: 'index.ts', - hash: 'hash2', - }, - ], - }, - createCache({ + nonProjectFiles: [], projectFileMap: { mylib: [ { @@ -172,49 +174,56 @@ describe('nx deps utils', () => { hash: 'hash1', }, ], + secondlib: [ + { + file: 'index.ts', + hash: 'hash2', + }, + ], + }, + }, + createCache({ + fileMap: { + nonProjectFiles: [], + projectFileMap: { + mylib: [ + { + file: 'index.ts', + hash: 'hash1', + }, + ], + }, }, }) ); expect(r.filesToProcess).toEqual({ - secondlib: [ - { - file: 'index.ts', - hash: 'hash2', - }, - ], + projectFileMap: { + secondlib: [ + { + file: 'index.ts', + hash: 'hash2', + }, + ], + }, + nonProjectFiles: [], }); expect(r.cachedFileData).toEqual({ - mylib: { - 'index.ts': { - file: 'index.ts', - hash: 'hash1', + nonProjectFiles: {}, + projectFileMap: { + mylib: { + 'index.ts': { + file: 'index.ts', + hash: 'hash1', + }, }, }, }); - expect(r.filesToProcess).toEqual({ - secondlib: [{ file: 'index.ts', hash: 'hash2' }], - }); }); it('should handle cases when files change', () => { const r = extractCachedFileData( { - mylib: [ - { - file: 'index1.ts', - hash: 'hash1', - }, - { - file: 'index2.ts', - hash: 'hash2b', - }, - { - file: 'index4.ts', - hash: 'hash4', - }, - ], - }, - createCache({ + nonProjectFiles: [], projectFileMap: { mylib: [ { @@ -223,33 +232,60 @@ describe('nx deps utils', () => { }, { file: 'index2.ts', - hash: 'hash2', + hash: 'hash2b', }, { - file: 'index3.ts', - hash: 'hash3', + file: 'index4.ts', + hash: 'hash4', }, ], }, + }, + createCache({ + fileMap: { + nonProjectFiles: [], + projectFileMap: { + mylib: [ + { + file: 'index1.ts', + hash: 'hash1', + }, + { + file: 'index2.ts', + hash: 'hash2', + }, + { + file: 'index3.ts', + hash: 'hash3', + }, + ], + }, + }, }) ); expect(r.filesToProcess).toEqual({ - mylib: [ - { - file: 'index2.ts', - hash: 'hash2b', - }, - { - file: 'index4.ts', - hash: 'hash4', - }, - ], + nonProjectFiles: [], + projectFileMap: { + mylib: [ + { + file: 'index2.ts', + hash: 'hash2b', + }, + { + file: 'index4.ts', + hash: 'hash4', + }, + ], + }, }); expect(r.cachedFileData).toEqual({ - mylib: { - 'index1.ts': { - file: 'index1.ts', - hash: 'hash1', + nonProjectFiles: {}, + projectFileMap: { + mylib: { + 'index1.ts': { + file: 'index1.ts', + hash: 'hash1', + }, }, }, }); @@ -273,8 +309,8 @@ describe('nx deps utils', () => { }); }); - function createCache(p: Partial): ProjectFileMapCache { - const defaults: ProjectFileMapCache = { + function createCache(p: Partial): FileMapCache { + const defaults: FileMapCache = { version: '6.0', nxVersion: nxVersion, deps: {}, @@ -282,8 +318,11 @@ describe('nx deps utils', () => { mylib: ['libs/mylib/index.ts'], }, nxJsonPlugins: [{ name: 'plugin', version: '1.0.0' }], - projectFileMap: { - mylib: [], + fileMap: { + nonProjectFiles: [], + projectFileMap: { + mylib: [], + }, }, }; return { ...defaults, ...p }; diff --git a/packages/nx/src/project-graph/nx-deps-cache.ts b/packages/nx/src/project-graph/nx-deps-cache.ts index e0e47a7d2a94a..2cdc29eb15ebb 100644 --- a/packages/nx/src/project-graph/nx-deps-cache.ts +++ b/packages/nx/src/project-graph/nx-deps-cache.ts @@ -5,6 +5,7 @@ import { performance } from 'perf_hooks'; import { NxJsonConfiguration } from '../config/nx-json'; import { FileData, + FileMap, ProjectFileMap, ProjectGraph, } from '../config/project-graph'; @@ -18,14 +19,14 @@ import { } from '../utils/fileutils'; import { nxVersion } from '../utils/versions'; -export interface ProjectFileMapCache { +export interface FileMapCache { version: string; nxVersion: string; deps: Record; pathMappings: Record; nxJsonPlugins: { name: string; version: string }[]; pluginsConfig?: any; - projectFileMap: ProjectFileMap; + fileMap: FileMap; } export const nxProjectGraph = join( @@ -58,7 +59,7 @@ export function ensureCacheDirectory(): void { } } -export function readProjectFileMapCache(): null | ProjectFileMapCache { +export function readFileMapCache(): null | FileMapCache { performance.mark('read cache:start'); ensureCacheDirectory(); @@ -107,14 +108,14 @@ export function readProjectGraphCache(): null | ProjectGraph { export function createProjectFileMapCache( nxJson: NxJsonConfiguration<'*' | string[]>, packageJsonDeps: Record, - projectFileMap: ProjectFileMap, + fileMap: FileMap, tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } } ) { const nxJsonPlugins = (nxJson?.plugins || []).map((p) => ({ name: p, version: packageJsonDeps[p], })); - const newValue: ProjectFileMapCache = { + const newValue: FileMapCache = { version: '6.0', nxVersion: nxVersion, deps: packageJsonDeps, // TODO(v18): We can remove this in favor of nxVersion @@ -122,13 +123,13 @@ export function createProjectFileMapCache( pathMappings: tsConfig?.compilerOptions?.paths || {}, nxJsonPlugins, pluginsConfig: nxJson?.pluginsConfig, - projectFileMap, + fileMap, }; return newValue; } export function writeCache( - cache: ProjectFileMapCache, + cache: FileMapCache, projectGraph: ProjectGraph ): void { performance.mark('write cache:start'); @@ -169,7 +170,7 @@ export function writeCache( } export function shouldRecomputeWholeGraph( - cache: ProjectFileMapCache, + cache: FileMapCache, packageJsonDeps: Record, projects: Record, nxJson: NxJsonConfiguration, @@ -183,7 +184,7 @@ export function shouldRecomputeWholeGraph( } // we have a cached project that is no longer present - const cachedNodes = Object.keys(cache.projectFileMap); + const cachedNodes = Object.keys(cache.fileMap.projectFileMap); if (cachedNodes.some((p) => projects[p] === undefined)) { return true; } @@ -230,48 +231,84 @@ export function shouldRecomputeWholeGraph( return false; } +type CachedFileData = { + nonProjectFiles: Record; + projectFileMap: { [project: string]: Record }; +}; + /* This can only be invoked when the list of projects is either the same or new projects have been added, so every project in the cache has a corresponding project in fileMap */ export function extractCachedFileData( - fileMap: ProjectFileMap, - c: ProjectFileMapCache + fileMap: FileMap, + c: FileMapCache ): { - filesToProcess: ProjectFileMap; - cachedFileData: { [project: string]: { [file: string]: FileData } }; + filesToProcess: FileMap; + cachedFileData: CachedFileData; } { - const filesToProcess: ProjectFileMap = {}; - const cachedFileData: Record> = {}; - const currentProjects = Object.keys(fileMap).filter( - (name) => fileMap[name].length > 0 + const filesToProcess: FileMap = { + nonProjectFiles: [], + projectFileMap: {}, + }; + const cachedFileData: CachedFileData = { + nonProjectFiles: {}, + projectFileMap: {}, + }; + + const currentProjects = Object.keys(fileMap.projectFileMap).filter( + (name) => fileMap.projectFileMap[name].length > 0 ); currentProjects.forEach((p) => { processProjectNode( p, - c.projectFileMap, - cachedFileData, - filesToProcess, + c.fileMap.projectFileMap, + cachedFileData.projectFileMap, + filesToProcess.projectFileMap, fileMap ); }); + processNonProjectFiles( + c.fileMap.nonProjectFiles, + fileMap.nonProjectFiles, + filesToProcess.nonProjectFiles, + cachedFileData.nonProjectFiles + ); + return { filesToProcess, cachedFileData, }; } +function processNonProjectFiles( + cachedFiles: FileData[], + nonProjectFiles: FileData[], + filesToProcess: FileMap['nonProjectFiles'], + cachedFileData: CachedFileData['nonProjectFiles'] +) { + const cachedHashMap = new Map(cachedFiles.map((f) => [f.file, f])); + for (const f of nonProjectFiles) { + const cachedHash = cachedHashMap.get(f.file)?.hash; + if (!cachedHash || cachedHash !== f.hash) { + filesToProcess.push(f); + } else { + cachedFileData[f.file] = f; + } + } +} + function processProjectNode( projectName: string, cachedFileMap: ProjectFileMap, cachedFileData: { [project: string]: { [file: string]: FileData } }, filesToProcess: ProjectFileMap, - fileMap: ProjectFileMap + { projectFileMap }: FileMap ) { if (!cachedFileMap[projectName]) { - filesToProcess[projectName] = fileMap[projectName]; + filesToProcess[projectName] = projectFileMap[projectName]; return; } @@ -284,7 +321,7 @@ function processProjectNode( cachedFileData[projectName] = {}; } - for (let f of fileMap[projectName]) { + for (let f of projectFileMap[projectName]) { const fromCache = fileDataFromCache[f.file]; if (fromCache && fromCache.hash == f.hash) { cachedFileData[projectName][f.file] = fromCache; diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index fdea1e14d0457..a666a5c9b719d 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -13,7 +13,7 @@ import { } from '../config/project-graph'; import { ProjectConfiguration } from '../config/workspace-json-project-json'; import { CreateDependenciesContext } from '../utils/nx-plugin'; -import { getProjectFileMap } from './build-project-graph'; +import { getFileMap } from './build-project-graph'; /** * A class which builds up a project graph @@ -22,19 +22,19 @@ import { getProjectFileMap } from './build-project-graph'; export class ProjectGraphBuilder { // TODO(FrozenPandaz): make this private readonly graph: ProjectGraph; - private readonly fileMap: ProjectFileMap; + private readonly projectFileMap: ProjectFileMap; readonly removedEdges: { [source: string]: Set } = {}; constructor(graph?: ProjectGraph, fileMap?: ProjectFileMap) { if (graph) { this.graph = graph; - this.fileMap = fileMap || getProjectFileMap().projectFileMap; + this.projectFileMap = fileMap || getFileMap().fileMap?.projectFileMap; } else { this.graph = { nodes: {}, externalNodes: {}, dependencies: {}, }; - this.fileMap = fileMap || {}; + this.projectFileMap = fileMap || {}; } } @@ -256,7 +256,10 @@ export class ProjectGraphBuilder { }, { externalNodes: this.graph.externalNodes, - fileMap: this.fileMap, + fileMap: { + projectFileMap: { ...this.projectFileMap }, + nonProjectFiles: [], + }, // the validators only really care about the keys on this. projects: this.graph.nodes as any, filesToProcess: null, @@ -277,7 +280,7 @@ export class ProjectGraphBuilder { source, sourceFile, this.graph.nodes, - this.fileMap + this.projectFileMap ); if (!fileData.deps) { @@ -328,7 +331,7 @@ export class ProjectGraphBuilder { sourceProject: string ): Map> { const fileDeps = new Map>(); - const files = this.fileMap[sourceProject] || []; + const files = this.projectFileMap[sourceProject] || []; if (!files) { return fileDeps; } @@ -489,7 +492,7 @@ function validateCommonDependencyRules( } if ('sourceFile' in d && d.sourceFile) { // Throws if source file is not a valid file within the source project. - getFileData(d.source, d.sourceFile, projects, fileMap); + getFileData(d.source, d.sourceFile, projects, fileMap.projectFileMap); } } diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 7004f15d023a4..d0ba52be6c33a 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -1,7 +1,4 @@ -import { - readProjectFileMapCache, - readProjectGraphCache, -} from './nx-deps-cache'; +import { readFileMapCache, readProjectGraphCache } from './nx-deps-cache'; import { buildProjectGraphUsingProjectFileMap } from './build-project-graph'; import { output } from '../utils/output'; import { markDaemonAsDisabled, writeDaemonLogs } from '../daemon/tmp-dir'; @@ -74,21 +71,17 @@ export function readProjectsConfigurationFromProjectGraph( export async function buildProjectGraphWithoutDaemon() { const nxJson = readNxJson(); - const { - allWorkspaceFiles, - projectFileMap, - projectConfigurations, - externalNodes, - } = await retrieveWorkspaceFiles(workspaceRoot, nxJson); + const { allWorkspaceFiles, fileMap, projectConfigurations, externalNodes } = + await retrieveWorkspaceFiles(workspaceRoot, nxJson); const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false'; return ( await buildProjectGraphUsingProjectFileMap( projectConfigurations.projects, externalNodes, - projectFileMap, + fileMap, allWorkspaceFiles, - cacheEnabled ? readProjectFileMapCache() : null, + cacheEnabled ? readFileMapCache() : null, cacheEnabled ) ).projectGraph; diff --git a/packages/nx/src/project-graph/utils/implicit-project-dependencies.spec.ts b/packages/nx/src/project-graph/utils/implicit-project-dependencies.spec.ts index baafea5b774e0..ccabbe255ef7b 100644 --- a/packages/nx/src/project-graph/utils/implicit-project-dependencies.spec.ts +++ b/packages/nx/src/project-graph/utils/implicit-project-dependencies.spec.ts @@ -26,10 +26,7 @@ describe('implicit project dependencies', () => { applyImplicitDependencies( { - version: 2, - projects: { - proj1: { root: '', implicitDependencies: ['proj2'] }, - }, + proj1: { root: '', implicitDependencies: ['proj2'] }, }, builder ); @@ -57,10 +54,7 @@ describe('implicit project dependencies', () => { applyImplicitDependencies( { - version: 2, - projects: { - proj1: { root: '', implicitDependencies: ['!proj2'] }, - }, + proj1: { root: '', implicitDependencies: ['!proj2'] }, }, builder ); diff --git a/packages/nx/src/project-graph/utils/implicit-project-dependencies.ts b/packages/nx/src/project-graph/utils/implicit-project-dependencies.ts index 53d3e131ea7d5..6a294bfaa470b 100644 --- a/packages/nx/src/project-graph/utils/implicit-project-dependencies.ts +++ b/packages/nx/src/project-graph/utils/implicit-project-dependencies.ts @@ -1,12 +1,12 @@ -import { ProjectsConfigurations } from '../../config/workspace-json-project-json'; +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; import { ProjectGraphBuilder } from '../project-graph-builder'; export function applyImplicitDependencies( - projectsConfigurations: ProjectsConfigurations, + projects: Record, builder: ProjectGraphBuilder ) { - Object.keys(projectsConfigurations.projects).forEach((source) => { - const p = projectsConfigurations.projects[source]; + Object.keys(projects).forEach((source) => { + const p = projects[source]; if (p.implicitDependencies && p.implicitDependencies.length > 0) { p.implicitDependencies.forEach((target) => { if (target.startsWith('!')) { diff --git a/packages/nx/src/project-graph/utils/normalize-project-nodes.ts b/packages/nx/src/project-graph/utils/normalize-project-nodes.ts index 20d7643164e65..ff9eba7608591 100644 --- a/packages/nx/src/project-graph/utils/normalize-project-nodes.ts +++ b/packages/nx/src/project-graph/utils/normalize-project-nodes.ts @@ -15,18 +15,19 @@ import { readTargetDefaultsForTarget, resolveNxTokensInOptions, } from '../utils/project-configuration-utils'; +import { CreateDependenciesContext } from '../../utils/nx-plugin'; export async function normalizeProjectNodes( - ctx: ProjectGraphProcessorContext, + ctx: CreateDependenciesContext, builder: ProjectGraphBuilder, nxJson: NxJsonConfiguration ) { const toAdd = []; - const projects = Object.keys(ctx.projectsConfigurations.projects); + const projects = Object.keys(ctx.projects); // Used for expanding implicit dependencies (e.g. `@proj/*` or `tag:foo`) const partialProjectGraphNodes = projects.reduce((graph, project) => { - const projectConfiguration = ctx.projectsConfigurations.projects[project]; + const projectConfiguration = ctx.projects[project]; graph[project] = { name: project, type: projectConfiguration.projectType === 'library' ? 'lib' : 'app', // missing fallback to `e2e` @@ -38,7 +39,7 @@ export async function normalizeProjectNodes( }, {} as Record); for (const key of projects) { - const p = ctx.projectsConfigurations.projects[key]; + const p = ctx.projects[key]; p.implicitDependencies = normalizeImplicitDependencies( key, @@ -55,7 +56,7 @@ export async function normalizeProjectNodes( ? 'e2e' : 'app' : 'lib'; - const tags = ctx.projectsConfigurations.projects?.[key]?.tags || []; + const tags = ctx.projects?.[key]?.tags || []; toAdd.push({ name: key, diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 775caadaa269e..e7ebbb8626f53 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -86,7 +86,10 @@ export async function retrieveWorkspaceFiles( return { allWorkspaceFiles: buildAllWorkspaceFiles(projectFileMap, globalFiles), - projectFileMap, + fileMap: { + projectFileMap, + nonProjectFiles: globalFiles, + }, projectConfigurations: { version: 2, projects: projectConfigurations, diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index ef2829b227438..f03fed086dd56 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -32,7 +32,7 @@ import { import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from '../hasher/hash-task'; import { daemonClient } from '../daemon/client/client'; import { StoreRunInformationLifeCycle } from './life-cycles/store-run-information-life-cycle'; -import { getProjectFileMap } from '../project-graph/build-project-graph'; +import { getFileMap } from '../project-graph/build-project-graph'; import { performance } from 'perf_hooks'; async function getTerminalOutputLifeCycle( @@ -233,9 +233,9 @@ export async function invokeTasksRunner({ if (daemonClient.enabled()) { hasher = new DaemonBasedTaskHasher(daemonClient, runnerOptions); } else { - const { projectFileMap, allWorkspaceFiles } = getProjectFileMap(); + const { fileMap, allWorkspaceFiles } = getFileMap(); hasher = new InProcessTaskHasher( - projectFileMap, + fileMap?.projectFileMap, allWorkspaceFiles, projectGraph, nxJson, diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts index 54ac18e378cc7..7c93cc91c9ae4 100644 --- a/packages/nx/src/utils/nx-plugin.ts +++ b/packages/nx/src/utils/nx-plugin.ts @@ -2,6 +2,7 @@ import { existsSync } from 'fs'; import * as path from 'path'; import { FileData, + FileMap, ProjectFileMap, ProjectGraph, ProjectGraphExternalNode, @@ -95,12 +96,12 @@ export interface CreateDependenciesContext { /** * All files in the workspace */ - readonly fileMap: ProjectFileMap; + readonly fileMap: FileMap; /** * Files changes since last invocation */ - readonly filesToProcess: ProjectFileMap; + readonly filesToProcess: FileMap; readonly workspaceRoot: string; } From 8480d22bbc701559085170949cbf487dfd42a96b Mon Sep 17 00:00:00 2001 From: AgentEnder Date: Mon, 18 Sep 2023 15:22:22 -0400 Subject: [PATCH 2/2] feat(core): restore cached dependencies from non-project-files --- docs/generated/devkit/FileData.md | 10 +- docs/generated/devkit/ProjectGraphBuilder.md | 18 ++- ...-operators-imports-to-ngrx-router-store.ts | 4 +- .../src/rules/dependency-checks.spec.ts | 3 +- .../rules/enforce-module-boundaries.spec.ts | 7 +- .../eslint-plugin/src/utils/graph-utils.ts | 2 +- .../src/utils/project-graph-utils.ts | 4 +- .../utils/package-json/update-package-json.ts | 4 +- packages/nx/src/config/project-graph.ts | 37 +++++- ...project-graph-incremental-recomputation.ts | 4 +- .../src/native/workspace/workspace_files.rs | 5 - .../build-dependencies/build-dependencies.ts | 2 +- .../src/project-graph/build-project-graph.ts | 29 +++-- .../nx/src/project-graph/nx-deps-cache.ts | 8 +- .../project-graph/project-graph-builder.ts | 115 +++++++++++++++--- .../utils/retrieve-workspace-files.ts | 2 +- packages/workspace/index.ts | 1 - 17 files changed, 191 insertions(+), 64 deletions(-) diff --git a/docs/generated/devkit/FileData.md b/docs/generated/devkit/FileData.md index d56c3ebad6cf2..bdb6b8f2c5b68 100644 --- a/docs/generated/devkit/FileData.md +++ b/docs/generated/devkit/FileData.md @@ -6,7 +6,7 @@ Some metadata about a file ### Properties -- [deps](../../devkit/documents/FileData#deps): (string | [string, string])[] +- [deps](../../devkit/documents/FileData#deps): FileDataDependency[] - [file](../../devkit/documents/FileData#file): string - [hash](../../devkit/documents/FileData#hash): string @@ -14,7 +14,13 @@ Some metadata about a file ### deps -• `Optional` **deps**: (`string` \| [`string`, `string`])[] +• `Optional` **deps**: `FileDataDependency`[] + +An array of dependencies. If an element is just a string, +the dependency is assumed to be a static dependency targetting +that string. If the element is a tuple with two elements, the first element +inside of it is the target project, with the second element being the type of dependency. +If the tuple has 3 elements, the first is preceded by a source. --- diff --git a/docs/generated/devkit/ProjectGraphBuilder.md b/docs/generated/devkit/ProjectGraphBuilder.md index 7e0669d961a8f..4d29b4b8e4f26 100644 --- a/docs/generated/devkit/ProjectGraphBuilder.md +++ b/docs/generated/devkit/ProjectGraphBuilder.md @@ -15,6 +15,7 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ### Properties - [graph](../../devkit/documents/ProjectGraphBuilder#graph): ProjectGraph +- [nonProjectFiles](../../devkit/documents/ProjectGraphBuilder#nonprojectfiles): FileData[] - [projectFileMap](../../devkit/documents/ProjectGraphBuilder#projectfilemap): ProjectFileMap - [removedEdges](../../devkit/documents/ProjectGraphBuilder#removededges): Object @@ -40,14 +41,15 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ ### constructor -• **new ProjectGraphBuilder**(`graph?`, `fileMap?`) +• **new ProjectGraphBuilder**(`graph?`, `projectFileMap?`, `nonProjectFiles?`) #### Parameters -| Name | Type | -| :--------- | :-------------------------------------------------------- | -| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) | -| `fileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) | +| Name | Type | +| :----------------- | :-------------------------------------------------------- | +| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) | +| `projectFileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) | +| `nonProjectFiles?` | [`FileData`](../../devkit/documents/FileData)[] | ## Properties @@ -57,6 +59,12 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/ --- +### nonProjectFiles + +• `Private` `Readonly` **nonProjectFiles**: [`FileData`](../../devkit/documents/FileData)[] + +--- + ### projectFileMap • `Private` `Readonly` **projectFileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) diff --git a/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts b/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts index e9b138255245c..0697b000b59ef 100644 --- a/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts +++ b/packages/angular/src/migrations/update-16-2-0/switch-data-persistence-operators-imports-to-ngrx-router-store.ts @@ -9,7 +9,7 @@ import type { ImportDeclaration, ImportSpecifier, Node } from 'typescript'; import { FileChangeRecorder } from '../../utils/file-change-recorder'; import { ngrxVersion } from '../../utils/versions'; import { getProjectsFilteredByDependencies } from '../utils/projects'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; import { fileDataDepTarget } from 'nx/src/config/project-graph'; let tsquery: typeof import('@phenomnomnominal/tsquery').tsquery; @@ -35,7 +35,7 @@ export default async function (tree: Tree): Promise { ensureTypescript(); tsquery = require('@phenomnomnominal/tsquery').tsquery; - const cachedFileMap = readProjectFileMapCache().projectFileMap; + const cachedFileMap = readFileMapCache().fileMap.projectFileMap; const filesWithNxAngularImports: FileData[] = []; for (const { graphNode } of projects) { diff --git a/packages/eslint-plugin/src/rules/dependency-checks.spec.ts b/packages/eslint-plugin/src/rules/dependency-checks.spec.ts index fad6069fcc4ac..3c2f18a347d45 100644 --- a/packages/eslint-plugin/src/rules/dependency-checks.spec.ts +++ b/packages/eslint-plugin/src/rules/dependency-checks.spec.ts @@ -15,6 +15,7 @@ import { ProjectGraphExternalNode, } from '@nx/devkit'; import { Linter } from 'eslint'; +import { FileDataDependency } from 'nx/src/config/project-graph'; jest.mock('@nx/devkit', () => ({ ...jest.requireActual('@nx/devkit'), @@ -1652,7 +1653,7 @@ it('should require swc if @nx/js:swc executor', () => { expect(failures[0].line).toEqual(3); }); -function createFile(f: string, deps?: (string | [string, string])[]): FileData { +function createFile(f: string, deps?: FileDataDependency[]): FileData { return { file: f, hash: '', deps }; } diff --git a/packages/eslint-plugin/src/rules/enforce-module-boundaries.spec.ts b/packages/eslint-plugin/src/rules/enforce-module-boundaries.spec.ts index f07e863b1e60d..81615e529dc66 100644 --- a/packages/eslint-plugin/src/rules/enforce-module-boundaries.spec.ts +++ b/packages/eslint-plugin/src/rules/enforce-module-boundaries.spec.ts @@ -10,6 +10,7 @@ import enforceModuleBoundaries, { RULE_NAME as enforceModuleBoundariesRuleName, } from '../../src/rules/enforce-module-boundaries'; import { createProjectRootMappings } from 'nx/src/project-graph/utils/find-project-for-path'; +import { FileDataDependency } from 'nx/src/config/project-graph'; jest.mock('@nx/devkit', () => ({ ...jest.requireActual('@nx/devkit'), @@ -1303,8 +1304,8 @@ Violation detected in: { mylibName: [ createFile(`libs/mylib/src/main.ts`, [ - ['otherName', 'static'], - ['otherName', 'dynamic'], + ['otherName', DependencyType.static], + ['otherName', DependencyType.dynamic], ]), ], otherName: [createFile(`libs/other/index.ts`)], @@ -2250,7 +2251,7 @@ const baseConfig = { linter.defineParser('@typescript-eslint/parser', parser); linter.defineRule(enforceModuleBoundariesRuleName, enforceModuleBoundaries); -function createFile(f: string, deps?: (string | [string, string])[]): FileData { +function createFile(f: string, deps?: FileDataDependency[]): FileData { return { file: f, hash: '', deps }; } diff --git a/packages/eslint-plugin/src/utils/graph-utils.ts b/packages/eslint-plugin/src/utils/graph-utils.ts index 063bd96e7774a..bb4ef5107f192 100644 --- a/packages/eslint-plugin/src/utils/graph-utils.ts +++ b/packages/eslint-plugin/src/utils/graph-utils.ts @@ -149,7 +149,7 @@ export function findFilesInCircularPath( for (let i = 0; i < circularPath.length - 1; i++) { const next = circularPath[i + 1].name; - const files: FileData[] = projectFileMap[circularPath[i].name] || []; + const files = projectFileMap[circularPath[i].name] || []; filePathChain.push( files .filter( diff --git a/packages/eslint-plugin/src/utils/project-graph-utils.ts b/packages/eslint-plugin/src/utils/project-graph-utils.ts index b7ffe82af5329..27afc607e85d1 100644 --- a/packages/eslint-plugin/src/utils/project-graph-utils.ts +++ b/packages/eslint-plugin/src/utils/project-graph-utils.ts @@ -11,7 +11,7 @@ import { } from 'nx/src/project-graph/utils/find-project-for-path'; import { readNxJson } from 'nx/src/config/configuration'; import { TargetProjectLocator } from '@nx/js/src/internal'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; export function ensureGlobalProjectGraph(ruleName: string) { /** @@ -37,7 +37,7 @@ export function ensureGlobalProjectGraph(ruleName: string) { globalThis.projectRootMappings = createProjectRootMappings( projectGraph.nodes ); - globalThis.projectFileMap = readProjectFileMapCache().projectFileMap; + globalThis.projectFileMap = readFileMapCache().fileMap.projectFileMap; globalThis.targetProjectLocator = new TargetProjectLocator( projectGraph.nodes, projectGraph.externalNodes diff --git a/packages/js/src/utils/package-json/update-package-json.ts b/packages/js/src/utils/package-json/update-package-json.ts index 74cd9cb0f2fe1..41630fc895be2 100644 --- a/packages/js/src/utils/package-json/update-package-json.ts +++ b/packages/js/src/utils/package-json/update-package-json.ts @@ -25,7 +25,7 @@ import { writeFileSync } from 'fs-extra'; import { fileExists } from 'nx/src/utils/fileutils'; import type { PackageJson } from 'nx/src/utils/package-json'; import { existsSync } from 'fs'; -import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; +import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; import { getRelativeDirectoryToProjectRoot } from '../get-main-file-dir'; @@ -57,7 +57,7 @@ export function updatePackageJson( ): void { let packageJson: PackageJson; if (fileMap == null) { - fileMap = readProjectFileMapCache()?.projectFileMap || {}; + fileMap = readFileMapCache()?.fileMap?.projectFileMap || {}; } if (options.updateBuildableProjectDepsInPackageJson) { diff --git a/packages/nx/src/config/project-graph.ts b/packages/nx/src/config/project-graph.ts index 14b90a7e1f65b..8b5f7dd4eefae 100644 --- a/packages/nx/src/config/project-graph.ts +++ b/packages/nx/src/config/project-graph.ts @@ -11,15 +11,42 @@ import { NxJsonConfiguration } from './nx-json'; export interface FileData { file: string; hash: string; - deps?: (string | [string, string])[]; + /** + * An array of dependencies. If an element is just a string, + * the dependency is assumed to be a static dependency targetting + * that string. If the element is a tuple with two elements, the first element + * inside of it is the target project, with the second element being the type of dependency. + * If the tuple has 3 elements, the first is preceded by a source. + */ + deps?: FileDataDependency[]; } -export function fileDataDepTarget(dep: string | [string, string]) { - return typeof dep === 'string' ? dep : dep[0]; +/** + * A file data dependency, as stored in the cache. If just a string, + * the dependency is assumed to be a static dependency targetting + * that string. If it is a tuple with two elements, the first element + * inside of it is the target project, with the second element being the type of dependency. + * If the tuple has 3 elements, the first is preceded by a source. + */ +export type FileDataDependency = + | string + | [target: string, type: DependencyType] + | [source: string, target: string, type: DependencyType]; + +export function fileDataDepTarget(dep: FileDataDependency) { + return typeof dep === 'string' + ? dep + : Array.isArray(dep) && dep.length === 2 + ? dep[0] + : dep[1]; } -export function fileDataDepType(dep: string | [string, string]) { - return typeof dep === 'string' ? 'static' : dep[1]; +export function fileDataDepType(dep: FileDataDependency) { + return typeof dep === 'string' + ? 'static' + : Array.isArray(dep) && dep.length === 2 + ? dep[1] + : dep[2]; } export interface FileMap { diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index fb6bf04b75a16..915361d631ad3 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -247,7 +247,7 @@ async function processFilesAndCreateAndSerializeProjectGraph() { } } -function copyFileData(d: FileData[]) { +function copyFileData(d: T[]) { return d.map((t) => ({ ...t })); } @@ -257,7 +257,7 @@ function copyFileMap(m: FileMap) { projectFileMap: {}, }; for (let p of Object.keys(m.projectFileMap)) { - c[p] = copyFileData(m.projectFileMap[p]); + c.projectFileMap[p] = copyFileData(m.projectFileMap[p]); } return c; } diff --git a/packages/nx/src/native/workspace/workspace_files.rs b/packages/nx/src/native/workspace/workspace_files.rs index eeae027316227..891b1fe1302c1 100644 --- a/packages/nx/src/native/workspace/workspace_files.rs +++ b/packages/nx/src/native/workspace/workspace_files.rs @@ -20,11 +20,6 @@ pub struct NxWorkspaceFiles { pub external_nodes: HashMap, } -pub struct FileMap { - pub project_file_map: HashMap>, - pub non_project_files: Vec -} - pub(super) fn get_files( globs: Vec, parse_configurations: ConfigurationParser, diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts index d5d7af7a883e0..09d8da40396e6 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts @@ -43,7 +43,7 @@ export function buildExplicitDependencies( function totalNumberOfFilesToProcess(ctx: CreateDependenciesContext) { let totalNumOfFilesToProcess = 0; - Object.values(ctx.filesToProcess).forEach( + Object.values(ctx.filesToProcess.projectFileMap).forEach( (t) => (totalNumOfFilesToProcess += t.length) ); return totalNumOfFilesToProcess; diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 36fc2a3fb2256..42d78112985a0 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -4,6 +4,7 @@ import { performance } from 'perf_hooks'; import { assertWorkspaceValidity } from '../utils/assert-workspace-validity'; import { FileData } from './file-utils'; import { + CachedFileData, createProjectFileMapCache, extractCachedFileData, FileMapCache, @@ -78,10 +79,10 @@ export async function buildProjectGraphUsingProjectFileMap( const packageJsonDeps = readCombinedDeps(); const rootTsConfig = readRootTsConfig(); - let filesToProcess; - let cachedFileData; + let filesToProcess: FileMap; + let cachedFileData: CachedFileData; const useCacheData = - fileMap && + fileMapCache && !shouldRecomputeWholeGraph( fileMapCache, packageJsonDeps, @@ -95,7 +96,10 @@ export async function buildProjectGraphUsingProjectFileMap( cachedFileData = fromCache.cachedFileData; } else { filesToProcess = fileMap; - cachedFileData = {}; + cachedFileData = { + nonProjectFiles: {}, + projectFileMap: {}, + }; } const context = createContext( @@ -155,7 +159,7 @@ async function buildProjectGraphUsingContext( nxJson: NxJsonConfiguration, knownExternalNodes: Record, ctx: CreateDependenciesContext, - cachedFileData: { [project: string]: { [file: string]: FileData } }, + cachedFileData: CachedFileData, projectGraphVersion: string ) { performance.mark('build project graph:start'); @@ -172,14 +176,20 @@ async function buildProjectGraphUsingContext( const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph); const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap); - for (const proj of Object.keys(cachedFileData)) { - for (const f of ctx.fileMap[proj] || []) { - const cached = cachedFileData[proj][f.file]; + for (const proj of Object.keys(cachedFileData.projectFileMap)) { + for (const f of ctx.fileMap.projectFileMap[proj] || []) { + const cached = cachedFileData.projectFileMap[proj][f.file]; if (cached && cached.deps) { f.deps = [...cached.deps]; } } } + for (const file of ctx.fileMap.nonProjectFiles) { + const cached = cachedFileData.nonProjectFiles[file.file]; + if (cached?.deps) { + file.deps = [...cached.deps]; + } + } applyImplicitDependencies(ctx.projects, updatedBuilder); @@ -267,7 +277,8 @@ async function updateProjectGraphWithPlugins( if (isNxPluginV2(plugin) && plugin.createDependencies) { const builder = new ProjectGraphBuilder( graph, - context.fileMap.projectFileMap + context.fileMap.projectFileMap, + context.fileMap.nonProjectFiles ); const newDependencies = await plugin.createDependencies(context); for (const targetProjectDependency of newDependencies) { diff --git a/packages/nx/src/project-graph/nx-deps-cache.ts b/packages/nx/src/project-graph/nx-deps-cache.ts index 2cdc29eb15ebb..bf5d2ac7a078a 100644 --- a/packages/nx/src/project-graph/nx-deps-cache.ts +++ b/packages/nx/src/project-graph/nx-deps-cache.ts @@ -231,7 +231,7 @@ export function shouldRecomputeWholeGraph( return false; } -type CachedFileData = { +export type CachedFileData = { nonProjectFiles: Record; projectFileMap: { [project: string]: Record }; }; @@ -291,11 +291,11 @@ function processNonProjectFiles( ) { const cachedHashMap = new Map(cachedFiles.map((f) => [f.file, f])); for (const f of nonProjectFiles) { - const cachedHash = cachedHashMap.get(f.file)?.hash; - if (!cachedHash || cachedHash !== f.hash) { + const cachedFile = cachedHashMap.get(f.file); + if (!cachedFile || cachedFile.hash !== f.hash) { filesToProcess.push(f); } else { - cachedFileData[f.file] = f; + cachedFileData[f.file] = cachedFile; } } } diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index a666a5c9b719d..aa09bfb5fdb0b 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -3,8 +3,11 @@ */ import { DependencyType, + FileData, + FileDataDependency, fileDataDepTarget, fileDataDepType, + FileMap, ProjectFileMap, ProjectGraph, ProjectGraphDependency, @@ -22,19 +25,34 @@ import { getFileMap } from './build-project-graph'; export class ProjectGraphBuilder { // TODO(FrozenPandaz): make this private readonly graph: ProjectGraph; + private readonly projectFileMap: ProjectFileMap; + private readonly nonProjectFiles: FileData[]; + readonly removedEdges: { [source: string]: Set } = {}; - constructor(graph?: ProjectGraph, fileMap?: ProjectFileMap) { + + constructor( + graph?: ProjectGraph, + projectFileMap?: ProjectFileMap, + nonProjectFiles?: FileMap['nonProjectFiles'] + ) { + if (!projectFileMap || !nonProjectFiles) { + const fileMap = getFileMap().fileMap; + projectFileMap ??= fileMap.projectFileMap; + nonProjectFiles ??= fileMap.nonProjectFiles; + } if (graph) { this.graph = graph; - this.projectFileMap = fileMap || getFileMap().fileMap?.projectFileMap; + this.projectFileMap = projectFileMap; + this.nonProjectFiles = nonProjectFiles; } else { this.graph = { nodes: {}, externalNodes: {}, dependencies: {}, }; - this.projectFileMap = fileMap || {}; + this.projectFileMap = projectFileMap || {}; + this.nonProjectFiles = nonProjectFiles || []; } } @@ -234,6 +252,28 @@ export class ProjectGraphBuilder { } } } + for (const file of this.nonProjectFiles) { + if (file.deps) { + for (const dep of file.deps) { + if (!Array.isArray(dep)) { + throw new Error( + 'Cached data on non project files should be a tuple' + ); + } + const [source, target, type] = dep; + if (!source || !target || !type) { + throw new Error( + 'Cached dependencies for non project files should be a tuple of length 3.' + ); + } + this.graph.dependencies[source].push({ + source, + target, + type, + }); + } + } + } return this.graph; } @@ -257,8 +297,8 @@ export class ProjectGraphBuilder { { externalNodes: this.graph.externalNodes, fileMap: { - projectFileMap: { ...this.projectFileMap }, - nonProjectFiles: [], + projectFileMap: this.projectFileMap, + nonProjectFiles: this.nonProjectFiles, }, // the validators only really care about the keys on this. projects: this.graph.nodes as any, @@ -276,23 +316,28 @@ export class ProjectGraphBuilder { ); if (sourceFile) { - const fileData = getFileData( + let fileData = getProjectFileData( source, sourceFile, - this.graph.nodes, this.projectFileMap ); + const isProjectFileData = !!fileData; + fileData ??= getNonProjectFileData(sourceFile, this.nonProjectFiles); if (!fileData.deps) { fileData.deps = []; } + if ( !fileData.deps.find( (t) => fileDataDepTarget(t) === target && fileDataDepType(t) === type ) ) { - const dep: string | [string, string] = - type === 'static' ? target : [target, type]; + const dep: FileDataDependency = isProjectFileData + ? type === 'static' + ? target + : [target, type] + : [source, target, type]; fileData.deps.push(dep); } } else if (!isDuplicate) { @@ -491,8 +536,19 @@ function validateCommonDependencyRules( throw new Error(`External projects can't depend on internal projects`); } if ('sourceFile' in d && d.sourceFile) { - // Throws if source file is not a valid file within the source project. - getFileData(d.source, d.sourceFile, projects, fileMap.projectFileMap); + if (projects[d.source]) { + // Throws if source file is not a valid file within the source project. + // We can pass empty array for all workspace files here, since its not checked by the impl. + // We need all workspace files in here for the TODO comment though, so lets figure that out. + getFileData( + d.source, + d.sourceFile, + projects, + externalNodes, + fileMap.projectFileMap, + fileMap.nonProjectFiles + ); + } } } @@ -531,21 +587,44 @@ function validateStaticDependency( } } -function getFileData( +function getProjectFileData( source: string, sourceFile: string, - projects: Record, fileMap: ProjectFileMap ) { - const sourceProject = projects[source]; - if (!sourceProject) { - throw new Error(`Source project is not a project node: ${sourceProject}`); + let fileData = (fileMap[source] || []).find((f) => f.file === sourceFile); + if (fileData) { + return fileData; } - const fileData = (fileMap[source] || []).find((f) => f.file === sourceFile); +} + +function getNonProjectFileData(sourceFile: string, files: FileData[]) { + const fileData = files.find((f) => f.file === sourceFile); if (!fileData) { throw new Error( - `Source project ${source} does not have a file: ${sourceFile}` + `Source file "${sourceFile}" does not exist in the workspace.` ); } return fileData; } + +function getFileData( + source: string, + sourceFile: string, + projects: Record, + externalNodes: Record, + fileMap: ProjectFileMap, + nonProjectFiles: FileData[] +) { + const sourceProject = projects[source]; + const matchingExternalNode = externalNodes[source]; + + if (!sourceProject && !matchingExternalNode) { + throw new Error(`Source project is not a project node: ${sourceProject}`); + } + + return ( + getProjectFileData(source, sourceFile, fileMap) ?? + getNonProjectFileData(sourceFile, nonProjectFiles) + ); +} diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index e7ebbb8626f53..9ab713def47d8 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -255,7 +255,7 @@ function buildAllWorkspaceFiles( globalFiles: FileData[] ): FileData[] { performance.mark('get-all-workspace-files:start'); - let fileData = Object.values(projectFileMap).flat(); + let fileData: FileData[] = Object.values(projectFileMap).flat(); fileData = fileData.concat(globalFiles); performance.mark('get-all-workspace-files:end'); diff --git a/packages/workspace/index.ts b/packages/workspace/index.ts index e506584ad0104..dd278a00fd4eb 100644 --- a/packages/workspace/index.ts +++ b/packages/workspace/index.ts @@ -15,7 +15,6 @@ export { readWorkspaceConfig, readPackageJson, } from 'nx/src/project-graph/file-utils'; -export { ProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache'; export { getWorkspacePath,