Skip to content

Commit

Permalink
feat(core): restore cached dependencies from non-project-files
Browse files Browse the repository at this point in the history
  • Loading branch information
AgentEnder committed Sep 19, 2023
1 parent 039a866 commit 3d2cb30
Show file tree
Hide file tree
Showing 15 changed files with 208 additions and 59 deletions.
10 changes: 8 additions & 2 deletions docs/generated/devkit/FileData.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,21 @@ Some metadata about a file

### Properties

- [deps](../../devkit/documents/FileData#deps): (string | [string, string])[]
- [deps](../../devkit/documents/FileData#deps): FileDataDependency[]
- [file](../../devkit/documents/FileData#file): string
- [hash](../../devkit/documents/FileData#hash): string

## Properties

### deps

`Optional` **deps**: (`string` \| [`string`, `string`])[]
`Optional` **deps**: `FileDataDependency`[]

An array of dependencies. If an element is just a string,
the dependency is assumed to be a static dependency targetting
that string. If the element is a tuple with two elements, the first element
inside of it is the target project, with the second element being the type of dependency.
If the tuple has 3 elements, the first is preceded by a source.

---

Expand Down
25 changes: 20 additions & 5 deletions docs/generated/devkit/ProjectGraphBuilder.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/

### Properties

- [allWorkspaceFiles](../../devkit/documents/ProjectGraphBuilder#allworkspacefiles): FileData[]
- [graph](../../devkit/documents/ProjectGraphBuilder#graph): ProjectGraph
- [nonProjectFiles](../../devkit/documents/ProjectGraphBuilder#nonprojectfiles): FileData[]
- [projectFileMap](../../devkit/documents/ProjectGraphBuilder#projectfilemap): ProjectFileMap
- [removedEdges](../../devkit/documents/ProjectGraphBuilder#removededges): Object

Expand All @@ -40,23 +42,36 @@ The ProjectGraphProcessor has been deprecated. Use a [CreateNodes](../../devkit/

### constructor

**new ProjectGraphBuilder**(`graph?`, `fileMap?`)
**new ProjectGraphBuilder**(`graph?`, `projectFileMap?`, `nonProjectFiles?`)

#### Parameters

| Name | Type |
| :--------- | :-------------------------------------------------------- |
| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) |
| `fileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) |
| Name | Type |
| :----------------- | :-------------------------------------------------------- |
| `graph?` | [`ProjectGraph`](../../devkit/documents/ProjectGraph) |
| `projectFileMap?` | [`ProjectFileMap`](../../devkit/documents/ProjectFileMap) |
| `nonProjectFiles?` | [`FileData`](../../devkit/documents/FileData)[] |

## Properties

### allWorkspaceFiles

`Private` `Readonly` **allWorkspaceFiles**: [`FileData`](../../devkit/documents/FileData)[]

---

### graph

`Readonly` **graph**: [`ProjectGraph`](../../devkit/documents/ProjectGraph)

---

### nonProjectFiles

`Private` `Readonly` **nonProjectFiles**: [`FileData`](../../devkit/documents/FileData)[]

---

### projectFileMap

`Private` `Readonly` **projectFileMap**: [`ProjectFileMap`](../../devkit/documents/ProjectFileMap)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import type { ImportDeclaration, ImportSpecifier, Node } from 'typescript';
import { FileChangeRecorder } from '../../utils/file-change-recorder';
import { ngrxVersion } from '../../utils/versions';
import { getProjectsFilteredByDependencies } from '../utils/projects';
import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache';
import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache';
import { fileDataDepTarget } from 'nx/src/config/project-graph';

let tsquery: typeof import('@phenomnomnominal/tsquery').tsquery;
Expand All @@ -35,7 +35,7 @@ export default async function (tree: Tree): Promise<void> {

ensureTypescript();
tsquery = require('@phenomnomnominal/tsquery').tsquery;
const cachedFileMap = readProjectFileMapCache().projectFileMap;
const cachedFileMap = readFileMapCache().fileMap.projectFileMap;

const filesWithNxAngularImports: FileData[] = [];
for (const { graphNode } of projects) {
Expand Down
3 changes: 2 additions & 1 deletion packages/eslint-plugin/src/rules/dependency-checks.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
ProjectGraphExternalNode,
} from '@nx/devkit';
import { Linter } from 'eslint';
import { FileDataDependency } from 'nx/src/config/project-graph';

jest.mock('@nx/devkit', () => ({
...jest.requireActual<any>('@nx/devkit'),
Expand Down Expand Up @@ -1652,7 +1653,7 @@ it('should require swc if @nx/js:swc executor', () => {
expect(failures[0].line).toEqual(3);
});

function createFile(f: string, deps?: (string | [string, string])[]): FileData {
function createFile(f: string, deps?: FileDataDependency[]): FileData {
return { file: f, hash: '', deps };
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import enforceModuleBoundaries, {
RULE_NAME as enforceModuleBoundariesRuleName,
} from '../../src/rules/enforce-module-boundaries';
import { createProjectRootMappings } from 'nx/src/project-graph/utils/find-project-for-path';
import { FileDataDependency } from 'nx/src/config/project-graph';

jest.mock('@nx/devkit', () => ({
...jest.requireActual<any>('@nx/devkit'),
Expand Down Expand Up @@ -1303,8 +1304,8 @@ Violation detected in:
{
mylibName: [
createFile(`libs/mylib/src/main.ts`, [
['otherName', 'static'],
['otherName', 'dynamic'],
['otherName', DependencyType.static],
['otherName', DependencyType.dynamic],
]),
],
otherName: [createFile(`libs/other/index.ts`)],
Expand Down Expand Up @@ -2250,7 +2251,7 @@ const baseConfig = {
linter.defineParser('@typescript-eslint/parser', parser);
linter.defineRule(enforceModuleBoundariesRuleName, enforceModuleBoundaries);

function createFile(f: string, deps?: (string | [string, string])[]): FileData {
function createFile(f: string, deps?: FileDataDependency[]): FileData {
return { file: f, hash: '', deps };
}

Expand Down
2 changes: 1 addition & 1 deletion packages/eslint-plugin/src/utils/graph-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ export function findFilesInCircularPath(

for (let i = 0; i < circularPath.length - 1; i++) {
const next = circularPath[i + 1].name;
const files: FileData[] = projectFileMap[circularPath[i].name] || [];
const files = projectFileMap[circularPath[i].name] || [];
filePathChain.push(
files
.filter(
Expand Down
4 changes: 2 additions & 2 deletions packages/eslint-plugin/src/utils/project-graph-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
} from 'nx/src/project-graph/utils/find-project-for-path';
import { readNxJson } from 'nx/src/config/configuration';
import { TargetProjectLocator } from '@nx/js/src/internal';
import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache';
import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache';

export function ensureGlobalProjectGraph(ruleName: string) {
/**
Expand All @@ -37,7 +37,7 @@ export function ensureGlobalProjectGraph(ruleName: string) {
globalThis.projectRootMappings = createProjectRootMappings(
projectGraph.nodes
);
globalThis.projectFileMap = readProjectFileMapCache().projectFileMap;
globalThis.projectFileMap = readFileMapCache().fileMap.projectFileMap;
globalThis.targetProjectLocator = new TargetProjectLocator(
projectGraph.nodes,
projectGraph.externalNodes
Expand Down
4 changes: 2 additions & 2 deletions packages/js/src/utils/package-json/update-package-json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { writeFileSync } from 'fs-extra';
import { fileExists } from 'nx/src/utils/fileutils';
import type { PackageJson } from 'nx/src/utils/package-json';
import { existsSync } from 'fs';
import { readProjectFileMapCache } from 'nx/src/project-graph/nx-deps-cache';
import { readFileMapCache } from 'nx/src/project-graph/nx-deps-cache';

import { getRelativeDirectoryToProjectRoot } from '../get-main-file-dir';

Expand Down Expand Up @@ -57,7 +57,7 @@ export function updatePackageJson(
): void {
let packageJson: PackageJson;
if (fileMap == null) {
fileMap = readProjectFileMapCache()?.projectFileMap || {};
fileMap = readFileMapCache()?.fileMap?.projectFileMap || {};
}

if (options.updateBuildableProjectDepsInPackageJson) {
Expand Down
37 changes: 32 additions & 5 deletions packages/nx/src/config/project-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,42 @@ import { NxJsonConfiguration } from './nx-json';
export interface FileData {
file: string;
hash: string;
deps?: (string | [string, string])[];
/**
* An array of dependencies. If an element is just a string,
* the dependency is assumed to be a static dependency targetting
* that string. If the element is a tuple with two elements, the first element
* inside of it is the target project, with the second element being the type of dependency.
* If the tuple has 3 elements, the first is preceded by a source.
*/
deps?: FileDataDependency[];
}

export function fileDataDepTarget(dep: string | [string, string]) {
return typeof dep === 'string' ? dep : dep[0];
/**
* A file data dependency, as stored in the cache. If just a string,
* the dependency is assumed to be a static dependency targetting
* that string. If it is a tuple with two elements, the first element
* inside of it is the target project, with the second element being the type of dependency.
* If the tuple has 3 elements, the first is preceded by a source.
*/
export type FileDataDependency =
| string
| [target: string, type: DependencyType]
| [source: string, target: string, type: DependencyType];

export function fileDataDepTarget(dep: FileDataDependency) {
return typeof dep === 'string'
? dep
: Array.isArray(dep) && dep.length === 2
? dep[0]
: dep[1];
}

export function fileDataDepType(dep: string | [string, string]) {
return typeof dep === 'string' ? 'static' : dep[1];
export function fileDataDepType(dep: FileDataDependency) {
return typeof dep === 'string'
? 'static'
: Array.isArray(dep) && dep.length === 2
? dep[1]
: dep[2];
}

export interface FileMap {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ async function processFilesAndCreateAndSerializeProjectGraph() {
}
}

function copyFileData(d: FileData[]) {
function copyFileData<T extends FileData>(d: T[]) {
return d.map((t) => ({ ...t }));
}

Expand All @@ -257,7 +257,7 @@ function copyFileMap(m: FileMap) {
projectFileMap: {},
};
for (let p of Object.keys(m.projectFileMap)) {
c[p] = copyFileData(m.projectFileMap[p]);
c.projectFileMap[p] = copyFileData(m.projectFileMap[p]);
}
return c;
}
Expand Down
26 changes: 18 additions & 8 deletions packages/nx/src/project-graph/build-project-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { performance } from 'perf_hooks';
import { assertWorkspaceValidity } from '../utils/assert-workspace-validity';
import { FileData } from './file-utils';
import {
CachedFileData,
createProjectFileMapCache,
extractCachedFileData,
FileMapCache,
Expand Down Expand Up @@ -78,10 +79,10 @@ export async function buildProjectGraphUsingProjectFileMap(
const packageJsonDeps = readCombinedDeps();
const rootTsConfig = readRootTsConfig();

let filesToProcess;
let cachedFileData;
let filesToProcess: FileMap;
let cachedFileData: CachedFileData;
const useCacheData =
fileMap &&
fileMapCache &&
!shouldRecomputeWholeGraph(
fileMapCache,
packageJsonDeps,
Expand All @@ -95,7 +96,10 @@ export async function buildProjectGraphUsingProjectFileMap(
cachedFileData = fromCache.cachedFileData;
} else {
filesToProcess = fileMap;
cachedFileData = {};
cachedFileData = {
nonProjectFiles: {},
projectFileMap: {},
};
}

const context = createContext(
Expand Down Expand Up @@ -155,7 +159,7 @@ async function buildProjectGraphUsingContext(
nxJson: NxJsonConfiguration,
knownExternalNodes: Record<string, ProjectGraphExternalNode>,
ctx: CreateDependenciesContext,
cachedFileData: { [project: string]: { [file: string]: FileData } },
cachedFileData: CachedFileData,
projectGraphVersion: string
) {
performance.mark('build project graph:start');
Expand All @@ -172,14 +176,20 @@ async function buildProjectGraphUsingContext(
const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph);

const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap);
for (const proj of Object.keys(cachedFileData)) {
for (const f of ctx.fileMap[proj] || []) {
const cached = cachedFileData[proj][f.file];
for (const proj of Object.keys(cachedFileData.projectFileMap)) {
for (const f of ctx.fileMap.projectFileMap[proj] || []) {
const cached = cachedFileData.projectFileMap[proj][f.file];
if (cached && cached.deps) {
f.deps = [...cached.deps];
}
}
}
for (const file of ctx.fileMap.nonProjectFiles) {
const cached = cachedFileData.nonProjectFiles[file.file];
if (cached?.deps) {
file.deps = [...cached.deps];
}
}

applyImplicitDependencies(ctx.projects, updatedBuilder);

Expand Down
8 changes: 4 additions & 4 deletions packages/nx/src/project-graph/nx-deps-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ export function shouldRecomputeWholeGraph(
return false;
}

type CachedFileData = {
export type CachedFileData = {
nonProjectFiles: Record<string, FileData>;
projectFileMap: { [project: string]: Record<string, FileData> };
};
Expand Down Expand Up @@ -291,11 +291,11 @@ function processNonProjectFiles(
) {
const cachedHashMap = new Map(cachedFiles.map((f) => [f.file, f]));
for (const f of nonProjectFiles) {
const cachedHash = cachedHashMap.get(f.file)?.hash;
if (!cachedHash || cachedHash !== f.hash) {
const cachedFile = cachedHashMap.get(f.file);
if (!cachedFile || cachedFile.hash !== f.hash) {
filesToProcess.push(f);
} else {
cachedFileData[f.file] = f;
cachedFileData[f.file] = cachedFile;
}
}
}
Expand Down
Loading

0 comments on commit 3d2cb30

Please sign in to comment.