From dd1baecc007cd169638425a722113f4202e2be9f Mon Sep 17 00:00:00 2001 From: Kris Selden Date: Tue, 27 Jul 2021 13:57:03 -0700 Subject: [PATCH] Deflate AdjustImportsOptions The babel-loader cache key hashes babel options which has the AdjustImportsOptions used multiple times and is very large. This PR moves this into a file ignoring caching for now since everything it moves is already memoized and only thing this affects is in-repo-addons and isDevelopingAddon true which is currently an issue. This doesn't address relocatedFiles which is very large because it is outside the spot that is memoized. --- packages/compat/src/compat-app.ts | 13 +++- packages/compat/src/resolver.ts | 39 ++++++---- packages/core/src/app.ts | 4 +- .../core/src/babel-plugin-adjust-imports.ts | 77 +++++++++++-------- 4 files changed, 85 insertions(+), 48 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 8765bd7eb..183fee90a 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -22,7 +22,7 @@ import walkSync from 'walk-sync'; import { join } from 'path'; import { JSDOM } from 'jsdom'; import { V1Config } from './v1-config'; -import { statSync, readdirSync } from 'fs'; +import { statSync, readdirSync, writeFileSync } from 'fs'; import Options, { optionsWithDefaults } from './options'; import CompatResolver from './resolver'; import { activePackageRules, PackageRules, expandModuleRules } from './dependency-rules'; @@ -36,6 +36,7 @@ import { pathExistsSync } from 'fs-extra'; import { tmpdir } from '@embroider/shared-internals'; import { Options as AdjustImportsOptions } from '@embroider/core/src/babel-plugin-adjust-imports'; import { getEmberExports } from '@embroider/core/src/load-ember-template-compiler'; + import semver from 'semver'; interface TreeNames { @@ -337,15 +338,23 @@ class CompatAppAdapter implements AppAdapter { podModulePrefix: this.podModulePrefix(), options: this.options, activePackageRules: this.activeRules(), - adjustImportsOptions: this.adjustImportsOptions(), + adjustImportsOptionsFile: this.adjustImportsOptionsFile(), }); } + @Memoize() + adjustImportsOptionsFile(): string { + let file = join(this.root, '_adjust_imports.json'); + writeFileSync(file, JSON.stringify(this.adjustImportsOptions())); + return file; + } + @Memoize() adjustImportsOptions(): AdjustImportsOptions { return this.makeAdjustImportOptions(true); } + // this gets serialized out by babel plugin and ast plugin private makeAdjustImportOptions(outer: boolean): AdjustImportsOptions { let renamePackages = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-packages'])); let renameModules = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-modules'])); diff --git a/packages/compat/src/resolver.ts b/packages/compat/src/resolver.ts index 8e2f60d2a..465805090 100644 --- a/packages/compat/src/resolver.ts +++ b/packages/compat/src/resolver.ts @@ -22,7 +22,7 @@ import Options from './options'; import { ResolvedDep } from '@embroider/core/src/resolver'; import { dasherize } from './dasherize-component-name'; import { makeResolverTransform } from './resolver-transform'; -import { pathExistsSync } from 'fs-extra'; +import { pathExistsSync, readFileSync } from 'fs-extra'; import resolve from 'resolve'; export interface ComponentResolution { @@ -110,15 +110,24 @@ function extractOptions(options: Required | ResolverOptions): ResolverO }; } -interface RehydrationParams { +interface RehydrationParamsBase { root: string; modulePrefix: string; podModulePrefix?: string; options: ResolverOptions; activePackageRules: ActivePackageRules[]; +} + +interface RehydrationParamsWithFile extends RehydrationParamsBase { + adjustImportsOptionsFile: string; +} + +interface RehydrationParamsWithOptions extends RehydrationParamsBase { adjustImportsOptions: AdjustImportsOptions; } +type RehydrationParams = RehydrationParamsWithFile | RehydrationParamsWithOptions; + export function rehydrate(params: RehydrationParams) { return new CompatResolver(params); } @@ -173,7 +182,7 @@ export default class CompatResolver implements Resolver { // "inherit" the rules that are attached to their corresonding JS module. if (absPath.endsWith('.hbs')) { let stem = absPath.slice(0, -4); - for (let ext of this.params.adjustImportsOptions.resolvableExtensions) { + for (let ext of this.adjustImportsOptions.resolvableExtensions) { if (ext !== '.hbs') { let rules = this.rules.components.get(stem + ext); if (rules) { @@ -189,6 +198,14 @@ export default class CompatResolver implements Resolver { return this.rules.ignoredComponents.includes(dasherizedName); } + @Memoize() + get adjustImportsOptions(): AdjustImportsOptions { + const { params } = this; + return 'adjustImportsOptionsFile' in params + ? JSON.parse(readFileSync(params.adjustImportsOptionsFile, 'utf8')) + : params.adjustImportsOptions; + } + @Memoize() private get rules() { if (!this.templateCompiler) { @@ -357,7 +374,7 @@ export default class CompatResolver implements Resolver { try { absPath = resolve.sync(path, { basedir: dirname(from), - extensions: this.params.adjustImportsOptions.resolvableExtensions, + extensions: this.adjustImportsOptions.resolvableExtensions, }); } catch (err) { return; @@ -372,14 +389,14 @@ export default class CompatResolver implements Resolver { @Memoize() private get resolvableExtensionsPattern() { - return extensionsPattern(this.params.adjustImportsOptions.resolvableExtensions); + return extensionsPattern(this.adjustImportsOptions.resolvableExtensions); } absPathToRuntimePath(absPath: string, owningPackage?: { root: string; name: string }) { let pkg = owningPackage || PackageCache.shared('embroider-stage3').ownerOfFile(absPath); if (pkg) { let packageRuntimeName = pkg.name; - for (let [runtimeName, realName] of Object.entries(this.params.adjustImportsOptions.renamePackages)) { + for (let [runtimeName, realName] of Object.entries(this.adjustImportsOptions.renamePackages)) { if (realName === packageRuntimeName) { packageRuntimeName = runtimeName; break; @@ -408,7 +425,7 @@ export default class CompatResolver implements Resolver { } private tryHelper(path: string, from: string): Resolution | null { - for (let extension of this.params.adjustImportsOptions.resolvableExtensions) { + for (let extension of this.adjustImportsOptions.resolvableExtensions) { let absPath = join(this.params.root, 'helpers', path) + extension; if (pathExistsSync(absPath)) { return { @@ -426,10 +443,6 @@ export default class CompatResolver implements Resolver { return null; } - get adjustImportsOptions() { - return this.params.adjustImportsOptions; - } - @Memoize() private get appPackage(): AppPackagePlaceholder { return { root: this.params.root, name: this.params.modulePrefix }; @@ -440,7 +453,7 @@ export default class CompatResolver implements Resolver { if (parts.length > 1 && parts[0].length > 0) { let cache = PackageCache.shared('embroider-stage3'); let packageName = parts[0]; - let renamed = this.params.adjustImportsOptions.renamePackages[packageName]; + let renamed = this.adjustImportsOptions.renamePackages[packageName]; if (renamed) { packageName = renamed; } @@ -462,7 +475,7 @@ export default class CompatResolver implements Resolver { // as a key into the rules, and we want to be able to find our rules when // checking from our own template (among other times). - let extensions = ['.hbs', ...this.params.adjustImportsOptions.resolvableExtensions.filter(e => e !== '.hbs')]; + let extensions = ['.hbs', ...this.adjustImportsOptions.resolvableExtensions.filter(e => e !== '.hbs')]; let componentModules = [] as string[]; diff --git a/packages/core/src/app.ts b/packages/core/src/app.ts index 7984baadf..4101403d7 100644 --- a/packages/core/src/app.ts +++ b/packages/core/src/app.ts @@ -111,6 +111,8 @@ export interface AppAdapter { // compatibility adjustImportsOptions(): AdjustImportsOptions; + adjustImportsOptionsFile(): string; + // The template preprocessor plugins that are configured in the app. htmlbarsPlugins(): TemplateCompilerPlugins; @@ -417,7 +419,7 @@ export class AppBuilder { } return [ require.resolve('./babel-plugin-adjust-imports'), - Object.assign({}, this.adapter.adjustImportsOptions(), { relocatedFiles }), + { adjustImportsOptionsFile: this.adapter.adjustImportsOptionsFile(), relocatedFiles }, ]; } diff --git a/packages/core/src/babel-plugin-adjust-imports.ts b/packages/core/src/babel-plugin-adjust-imports.ts index bf80ba73c..de4ae06bf 100644 --- a/packages/core/src/babel-plugin-adjust-imports.ts +++ b/packages/core/src/babel-plugin-adjust-imports.ts @@ -3,38 +3,43 @@ import { join, dirname, resolve } from 'path'; import type { NodePath } from '@babel/traverse'; import type * as t from '@babel/types'; import { PackageCache, Package, V2Package, explicitRelative } from '@embroider/shared-internals'; -import { outputFileSync } from 'fs-extra'; +import { outputFileSync, readFileSync } from 'fs-extra'; import { Memoize } from 'typescript-memoize'; import { compile } from './js-handlebars'; import { handleImportDeclaration } from './mini-modules-polyfill'; interface State { adjustFile: AdjustFile; - opts: { - renamePackages: { - [fromName: string]: string; - }; - renameModules: { - [fromName: string]: string; - }; - extraImports: { - absPath: string; - target: string; - runtimeName?: string; - }[]; - externalsDir: string; - activeAddons: { - [packageName: string]: string; - }; - relocatedFiles: { [relativePath: string]: string }; - resolvableExtensions: string[]; - emberNeedsModulesPolyfill: boolean; - }; + opts: Options | DeflatedOptions; +} + +export interface DeflatedOptions { + adjustImportsOptionsFile: string; + relocatedFiles: { [relativePath: string]: string }; } type BabelTypes = typeof t; -export type Options = State['opts']; +export interface Options { + renamePackages: { + [fromName: string]: string; + }; + renameModules: { + [fromName: string]: string; + }; + extraImports: { + absPath: string; + target: string; + runtimeName?: string; + }[]; + externalsDir: string; + activeAddons: { + [packageName: string]: string; + }; + relocatedFiles: { [relativePath: string]: string }; + resolvableExtensions: string[]; + emberNeedsModulesPolyfill: boolean; +} const packageCache = PackageCache.shared('embroider-stage3'); @@ -99,7 +104,7 @@ function adjustSpecifier(specifier: string, file: AdjustFile, opts: Options, isD return specifier; } -function handleRenaming(specifier: string, sourceFile: AdjustFile, opts: State['opts']) { +function handleRenaming(specifier: string, sourceFile: AdjustFile, opts: Options) { let packageName = getPackageName(specifier); if (!packageName) { return specifier; @@ -342,8 +347,9 @@ export default function main(babel: unknown) { visitor: { Program: { enter(path: NodePath, state: State) { + let opts = ensureOpts(state); state.adjustFile = new AdjustFile(path.hub.file.opts.filename, state.opts.relocatedFiles); - addExtraImports(t, path, state.opts.extraImports); + addExtraImports(t, path, opts.extraImports); }, exit(path: NodePath, state: State) { for (let child of path.get('body')) { @@ -356,7 +362,7 @@ export default function main(babel: unknown) { CallExpression(path: NodePath, state: State) { if (isImportSyncExpression(t, path) || isDynamicImportExpression(t, path)) { const [source] = path.get('arguments'); - let { opts } = state; + let opts = ensureOpts(state); let specifier = adjustSpecifier((source.node as any).value, state.adjustFile, opts, true); source.replaceWith(t.stringLiteral(specifier)); return; @@ -374,7 +380,7 @@ export default function main(babel: unknown) { ); } - let { opts } = state; + let opts = ensureOpts(state); const dependencies = path.node.arguments[1]; @@ -411,7 +417,7 @@ function rewriteTopLevelImport( path: NodePath, state: State ) { - let { opts } = state; + let opts = ensureOpts(state); const { source } = path.node; if (source === null || source === undefined) { return; @@ -435,11 +441,7 @@ function rewriteTopLevelImport( return join(__dirname, '..'); }; -function addExtraImports( - t: BabelTypes, - path: NodePath, - extraImports: Required['extraImports'] -) { +function addExtraImports(t: BabelTypes, path: NodePath, extraImports: Required['extraImports']) { let counter = 0; for (let { absPath, target, runtimeName } of extraImports) { if (absPath === path.hub.file.opts.filename) { @@ -491,6 +493,17 @@ class AdjustFile { } } +function ensureOpts(state: State): Options { + let { opts } = state; + if ('adjustImportsOptionsFile' in opts) { + let inflated = JSON.parse(readFileSync(opts.adjustImportsOptionsFile, 'utf8')); + inflated.relocatedFiles = opts.relocatedFiles; + state.opts = inflated; + return inflated; + } + return opts; +} + // we don't want to allow things that resolve only by accident that are likely // to break in other setups. For example: import your dependencies' // dependencies, or importing your own name from within a monorepo (which will