Merge pull request #1010 from embroider-build/template-compilation

template compilation improvements

packages/addon-dev/src/rollup-hbs-plugin.ts

@@ -1,7 +1,7 @@
 import { createFilter } from '@rollup/pluginutils';
 import type { Plugin } from 'rollup';
 import { readFileSync } from 'fs';
-const backtick = '`';
+import { hbsToJS } from '@embroider/shared-internals';
 
 export default function rollupHbsPlugin(): Plugin {
   const filter = createFilter('**/*.hbs');
@@ -38,9 +38,7 @@ export default function rollupHbsPlugin(): Plugin {
       }
 
       let input = readFileSync(originalId, 'utf8');
-      let code =
-        `import { hbs } from 'ember-cli-htmlbars';\n` +
-        `export default hbs${backtick}${input}${backtick};`;
+      let code = hbsToJS(input);
       return {
         code,
       };

packages/compat/src/audit.ts

@@ -1,7 +1,7 @@
 import { readFileSync, readJSONSync } from 'fs-extra';
 import { dirname, join, resolve as resolvePath } from 'path';
 import resolveModule from 'resolve';
-import { applyVariantToTemplateCompiler, AppMeta, explicitRelative } from '@embroider/core';
+import { AppMeta, explicitRelative, hbsToJS } from '@embroider/core';
 import { Memoize } from 'typescript-memoize';
 import chalk from 'chalk';
 import jsdom from 'jsdom';
@@ -17,7 +17,7 @@ import {
 } from './audit/babel-visitor';
 import { AuditBuildOptions, AuditOptions } from './audit/options';
 import { buildApp, BuildError, isBuildError } from './audit/build';
-import CompatResolver from './resolver';
+import { AuditMessage } from './resolver';
 
 const { JSDOM } = jsdom;
 
@@ -201,7 +201,7 @@ export class Audit {
 
     let audit = new this(dir, options);
     if (options['reuse-build']) {
-      if (!audit.meta.babel.isParallelSafe || !audit.meta['template-compiler'].isParallelSafe) {
+      if (!audit.meta.babel.isParallelSafe) {
         throw new BuildError(
           `You can't use the ${chalk.red(
             '--reuse-build'
@@ -244,33 +244,11 @@ export class Audit {
     let config = require(join(this.appDir, this.meta.babel.filename));
     config = Object.assign({}, config);
     config.plugins = config.plugins.filter((p: any) => !isMacrosPlugin(p));
+
     config.ast = true;
     return config;
   }
 
-  @Memoize()
-  private get templateSetup(): { compile: (filename: string, content: string) => string; resolver: CompatResolver } {
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    let templateCompiler = require(join(this.appDir, this.meta['template-compiler'].filename));
-    let resolver = templateCompiler.params.resolver as CompatResolver;
-
-    resolver.enableAuditMode();
-
-    let compile = applyVariantToTemplateCompiler(
-      { name: 'default', runtime: 'all', optimizeForProduction: false },
-      templateCompiler.compile
-    );
-    return { compile, resolver };
-  }
-
-  private get templateCompiler(): (filename: string, content: string) => string {
-    return this.templateSetup.compile;
-  }
-
-  private get templateResolver(): CompatResolver {
-    return this.templateSetup.resolver;
-  }
-
   private debug(message: string, ...args: any[]) {
     if (this.options.debug) {
       console.log(message, ...args);
@@ -331,16 +309,31 @@ export class Audit {
   }
 
   async run(): Promise<AuditResults> {
-    this.debug(`meta`, this.meta);
-    for (let asset of this.meta.assets) {
-      if (asset.endsWith('.html')) {
-        this.scheduleVisit(resolvePath(this.appDir, asset), { isRoot: true });
+    (globalThis as any).embroider_audit = this.handleResolverError.bind(this);
+
+    try {
+      this.debug(`meta`, this.meta);
+      for (let asset of this.meta.assets) {
+        if (asset.endsWith('.html')) {
+          this.scheduleVisit(resolvePath(this.appDir, asset), { isRoot: true });
+        }
       }
+      await this.drainQueue();
+      this.linkModules();
+      this.inspectModules();
+      return AuditResults.create(this.appDir, this.findings, this.modules);
+    } finally {
+      delete (globalThis as any).embroider_audit;
     }
-    await this.drainQueue();
-    this.linkModules();
-    this.inspectModules();
-    return AuditResults.create(this.appDir, this.findings, this.modules);
+  }
+
+  private handleResolverError(msg: AuditMessage) {
+    this.pushFinding({
+      message: msg.message,
+      filename: msg.filename,
+      detail: msg.detail,
+      codeFrame: this.frames.render(this.frames.forSource(msg.source)(msg)),
+    });
   }
 
   private linkModules() {
@@ -484,7 +477,7 @@ export class Audit {
         dependencies: result.imports.map(i => i.source),
       };
     } catch (err) {
-      if (err.code === 'BABEL_PARSE_ERROR') {
+      if (['BABEL_PARSE_ERROR', 'BABEL_TRANSFORM_ERROR'].includes(err.code)) {
         return [
           {
             filename,
@@ -503,26 +496,7 @@ export class Audit {
     content: Buffer | string
   ): Promise<ParsedInternalModule['parsed'] | Finding[]> {
     let rawSource = content.toString('utf8');
-    let js;
-    try {
-      js = this.templateCompiler(filename, rawSource);
-    } catch (err) {
-      return [
-        {
-          filename,
-          message: `failed to compile template`,
-          detail: err.toString().replace(filename, explicitRelative(this.appDir, filename)),
-        },
-      ];
-    }
-    for (let err of this.templateResolver.errorsIn(filename)) {
-      this.pushFinding({
-        filename,
-        message: err.message,
-        detail: err.detail,
-        codeFrame: this.frames.render(this.frames.forSource(rawSource)(err)),
-      });
-    }
+    let js = hbsToJS(rawSource);
     return this.visitJS(filename, js);
   }
 
@@ -547,7 +521,7 @@ export class Audit {
   }
 
   private async resolve(specifier: string, fromPath: string): Promise<string | ResolutionFailure | undefined> {
-    if (specifier === '@embroider/macros') {
+    if (['@embroider/macros', '@ember/template-factory'].includes(specifier)) {
       return;
     }
     try {

packages/compat/src/detect-babel-plugins.ts

@@ -39,3 +39,41 @@ export function isColocationPlugin(item: PluginItem): boolean {
 
   return pluginPath.includes(join('ember-cli-htmlbars', 'lib', 'colocated-babel-plugin', sep));
 }
+
+// tests for the classic ember-cli-htmlbars-inline-precompile babel plugin
+export function isInlinePrecompilePlugin(item: PluginItem) {
+  if (typeof item === 'string') {
+    return matchesSourceFile(item);
+  }
+  if (hasProperties(item) && (item as any)._parallelBabel) {
+    return matchesSourceFile((item as any)._parallelBabel.requireFile);
+  }
+  if (Array.isArray(item) && item.length > 0) {
+    if (typeof item[0] === 'string') {
+      return matchesSourceFile(item[0]);
+    }
+    if (hasProperties(item[0]) && (item[0] as any)._parallelBabel) {
+      return matchesSourceFile((item[0] as any)._parallelBabel.requireFile);
+    }
+  }
+  return false;
+}
+
+function matchesSourceFile(filename: string) {
+  return Boolean(htmlbarPathMatches.find(match => filename.endsWith(match)));
+}
+
+function hasProperties(item: any) {
+  return item && (typeof item === 'object' || typeof item === 'function');
+}
+
+const htmlbarPathMatches = [
+  ['htmlbars-inline-precompile', 'index.js'].join(sep),
+  ['htmlbars-inline-precompile', 'lib', 'require-from-worker.js'].join(sep),
+  ['htmlbars-inline-precompile', 'index'].join(sep),
+  ['htmlbars-inline-precompile', 'lib', 'require-from-worker'].join(sep),
+  ['ember-cli-htmlbars', 'index.js'].join(sep),
+  ['ember-cli-htmlbars', 'lib', 'require-from-worker.js'].join(sep),
+  ['ember-cli-htmlbars', 'index'].join(sep),
+  ['ember-cli-htmlbars', 'lib', 'require-from-worker'].join(sep),
+];

packages/compat/src/resolver-transform.ts

@@ -4,8 +4,8 @@ import type { ASTv1 } from '@glimmer/syntax';
 // This is the AST transform that resolves components, helpers and modifiers at build time
 // and puts them into `dependencies`.
 export function makeResolverTransform(resolver: Resolver) {
-  function resolverTransform({ filename }: { filename: string }) {
-    resolver.enter(filename);
+  function resolverTransform({ filename, contents }: { filename: string; contents: string }) {
+    resolver.enter(filename, contents);
 
     let scopeStack = new ScopeStack();
 

packages/compat/src/resolver.ts

@@ -151,10 +151,19 @@ export function rehydrate(params: RehydrationParams) {
   return new CompatResolver(params);
 }
 
+export interface AuditMessage {
+  message: string;
+  detail: string;
+  loc: Loc;
+  source: string;
+  filename: string;
+}
+
 export default class CompatResolver implements Resolver {
   private dependencies: Map<string, Resolution[]> = new Map();
   private templateCompiler: TemplateCompiler | undefined;
-  private auditMode = false;
+  private auditHandler: undefined | ((msg: AuditMessage) => void);
+  private currentContents: string | undefined;
 
   _parallelBabel: {
     requireFile: string;
@@ -169,9 +178,12 @@ export default class CompatResolver implements Resolver {
       buildUsing: 'rehydrate',
       params,
     };
+    if ((globalThis as any).embroider_audit) {
+      this.auditHandler = (globalThis as any).embroider_audit;
+    }
   }
 
-  enter(moduleName: string) {
+  enter(moduleName: string, contents: string) {
     let rules = this.findComponentRules(moduleName);
     let deps: Resolution[];
     if (rules?.dependsOnComponents) {
@@ -180,6 +192,7 @@ export default class CompatResolver implements Resolver {
       deps = [];
     }
     this.dependencies.set(moduleName, deps);
+    this.currentContents = contents;
   }
 
   private add(resolution: Resolution, from: string) {
@@ -351,29 +364,13 @@ export default class CompatResolver implements Resolver {
     }
   }
 
-  // called by our audit tool. Forces staticComponents, staticHelpers and staticModifiers
-  // to activate so we can audit their behavior, while making their errors silent
-  // until we can gather them up at the end of the build for the audit results.
-  enableAuditMode() {
-    this.auditMode = true;
-  }
-
-  errorsIn(moduleName: string): ResolutionFail[] {
-    let deps = this.dependencies.get(moduleName);
-    if (deps) {
-      return deps.filter(d => d.type === 'error') as ResolutionFail[];
-    } else {
-      return [];
-    }
-  }
-
   dependenciesOf(moduleName: string): ResolvedDep[] {
     let flatDeps: Map<string, ResolvedDep> = new Map();
     let deps = this.dependencies.get(moduleName);
     if (deps) {
       for (let dep of deps) {
         if (dep.type === 'error') {
-          if (!this.auditMode && !this.params.options.allowUnsafeDynamicComponents) {
+          if (!this.auditHandler && !this.params.options.allowUnsafeDynamicComponents) {
             let e: ResolverDependencyError = new Error(
               `${dep.message}: ${dep.detail} in ${humanReadableFile(this.params.root, moduleName)}`
             );
@@ -382,6 +379,15 @@ export default class CompatResolver implements Resolver {
             e.moduleName = moduleName;
             throw e;
           }
+          if (this.auditHandler) {
+            this.auditHandler({
+              message: dep.message,
+              filename: moduleName,
+              detail: dep.detail,
+              loc: dep.loc,
+              source: this.currentContents!,
+            });
+          }
         } else {
           for (let entry of dep.modules) {
             let { runtimeName } = entry;
@@ -441,15 +447,15 @@ export default class CompatResolver implements Resolver {
   }
 
   private get staticComponentsEnabled(): boolean {
-    return this.params.options.staticComponents || this.auditMode;
+    return this.params.options.staticComponents || Boolean(this.auditHandler);
   }
 
   private get staticHelpersEnabled(): boolean {
-    return this.params.options.staticHelpers || this.auditMode;
+    return this.params.options.staticHelpers || Boolean(this.auditHandler);
   }
 
   private get staticModifiersEnabled(): boolean {
-    return this.params.options.staticModifiers || this.auditMode;
+    return this.params.options.staticModifiers || Boolean(this.auditHandler);
   }
 
   private tryHelper(path: string, from: string): Resolution | null {

packages/compat/src/template-compiler-broccoli-plugin.ts

@@ -4,26 +4,19 @@ import type { TemplateCompiler } from '@embroider/core';
 import { join } from 'path';
 
 export default class TemplateCompileTree extends Filter {
-  constructor(inputTree: Node, private templateCompiler: TemplateCompiler, private stage: 1 | 3) {
+  constructor(inputTree: Node, private templateCompiler: TemplateCompiler) {
     super(inputTree, {
-      name: `embroider-template-compile-stage${stage}`,
+      name: `embroider-template-compile-stage1`,
       persist: true,
       extensions: ['hbs', 'handlebars'],
-      // in stage3 we are changing the file extensions from hbs to js. In
-      // stage1, we are just keeping hbs.
-      targetExtension: stage === 3 ? 'js' : undefined,
     });
   }
 
   processString(source: string, relativePath: string) {
-    if (this.stage === 1) {
-      return this.templateCompiler.applyTransforms(relativePath, source);
-    } else {
-      return this.templateCompiler.compile(relativePath, source);
-    }
+    return this.templateCompiler.applyTransforms(relativePath, source);
   }
   cacheKeyProcessString(source: string, relativePath: string) {
-    return `${this.stage}-${this.templateCompiler.cacheKey}` + super.cacheKeyProcessString(source, relativePath);
+    return `1-${this.templateCompiler.cacheKey}` + super.cacheKeyProcessString(source, relativePath);
   }
   baseDir() {
     return join(__dirname, '..');

packages/compat/src/v1-addon.ts

@@ -31,7 +31,12 @@ import V1App from './v1-app';
 import modulesCompat from './modules-compat';
 import writeFile from 'broccoli-file-creator';
 import SynthesizeTemplateOnlyComponents from './synthesize-template-only-components';
-import { isEmberAutoImportDynamic, isCompactReexports, isColocationPlugin } from './detect-babel-plugins';
+import {
+  isEmberAutoImportDynamic,
+  isCompactReexports,
+  isColocationPlugin,
+  isInlinePrecompilePlugin,
+} from './detect-babel-plugins';
 import { ResolvedDep } from '@embroider/core/src/resolver';
 import TemplateCompilerBroccoliPlugin from './template-compiler-broccoli-plugin';
 import { fromPairs } from 'lodash';
@@ -207,7 +212,7 @@ export default class V1Addon {
       _addon: this,
       toTree(this: { _addon: V1Addon }, tree: Node): Node {
         if (this._addon.templateCompiler) {
-          return new TemplateCompilerBroccoliPlugin(tree, this._addon.templateCompiler, 1);
+          return new TemplateCompilerBroccoliPlugin(tree, this._addon.templateCompiler);
         } else {
           // when there are no custom AST transforms, we don't need to do
           // anything at all.
@@ -1089,7 +1094,7 @@ function babelPluginAllowedInStage1(plugin: PluginItem) {
     return false;
   }
 
-  if (NodeTemplateCompiler.isInlinePrecompilePlugin(plugin)) {
+  if (isInlinePrecompilePlugin(plugin)) {
     // Similarly, the inline precompile plugin must not run in stage1. We
     // want all templates uncompiled. Instead, we will be adding our own
     // plugin that only runs custom AST transforms inside inline