[WIP] Support .d.ts files

Fixes denoland#1432

cli/BUILD.gn

@@ -110,6 +110,7 @@ ts_sources = [
   "../js/mock_builtin.js",
   "../js/net.ts",
   "../js/os.ts",
+  "../js/performance.ts",
   "../js/permissions.ts",
   "../js/plugins.d.ts",
   "../js/process.ts",
@@ -126,6 +127,7 @@ ts_sources = [
   "../js/text_encoding.ts",
   "../js/timers.ts",
   "../js/truncate.ts",
+  "../js/type_directives.ts",
   "../js/types.ts",
   "../js/url.ts",
   "../js/url_search_params.ts",
@@ -134,7 +136,6 @@ ts_sources = [
   "../js/window.ts",
   "../js/workers.ts",
   "../js/write_file.ts",
-  "../js/performance.ts",
   "../js/version.ts",
   "../js/xeval.ts",
   "../tsconfig.json",

js/compiler.ts

@@ -12,6 +12,7 @@ import { sendSync } from "./dispatch";
 import * as flatbuffers from "./flatbuffers";
 import * as os from "./os";
 import { TextDecoder, TextEncoder } from "./text_encoding";
+import { getMappedModuleName, parseTypeDirectives } from "./type_directives";
 import { assert, notImplemented } from "./util";
 import * as util from "./util";
 import { window } from "./window";
@@ -112,6 +113,7 @@ interface SourceFile {
   filename: string | undefined;
   mediaType: msg.MediaType;
   sourceCode: string | undefined;
+  typeDirectives?: Record<string, string>;
 }
 
 interface EmitResult {
@@ -121,7 +123,7 @@ interface EmitResult {
 
 /** Ops to Rust to resolve and fetch a modules meta data. */
 function fetchSourceFile(specifier: string, referrer: string): SourceFile {
-  util.log("compiler.fetchSourceFile", { specifier, referrer });
+  util.log("fetchSourceFile", { specifier, referrer });
   // Send FetchSourceFile message
   const builder = flatbuffers.createBuilder();
   const specifier_ = builder.createString(specifier);
@@ -148,7 +150,8 @@ function fetchSourceFile(specifier: string, referrer: string): SourceFile {
     moduleName: fetchSourceFileRes.moduleName() || undefined,
     filename: fetchSourceFileRes.filename() || undefined,
     mediaType: fetchSourceFileRes.mediaType(),
-    sourceCode
+    sourceCode,
+    typeDirectives: parseTypeDirectives(sourceCode)
   };
 }
 
@@ -170,7 +173,7 @@ function humanFileSize(bytes: number): string {
 
 /** Ops to rest for caching source map and compiled js */
 function cache(extension: string, moduleId: string, contents: string): void {
-  util.log("compiler.cache", moduleId);
+  util.log("cache", extension, moduleId);
   const builder = flatbuffers.createBuilder();
   const extension_ = builder.createString(extension);
   const moduleId_ = builder.createString(moduleId);
@@ -191,7 +194,7 @@ const encoder = new TextEncoder();
 function emitBundle(fileName: string, data: string): void {
   // For internal purposes, when trying to emit to `$deno$` just no-op
   if (fileName.startsWith("$deno$")) {
-    console.warn("skipping compiler.emitBundle", fileName);
+    console.warn("skipping emitBundle", fileName);
     return;
   }
   const encodedData = encoder.encode(data);
@@ -219,7 +222,7 @@ function getExtension(
 }
 
 class Host implements ts.CompilerHost {
-  extensionCache: Record<string, ts.Extension> = {};
+  private _extensionCache: Record<string, ts.Extension> = {};
 
   private readonly _options: ts.CompilerOptions = {
     allowJs: true,
@@ -234,23 +237,37 @@ class Host implements ts.CompilerHost {
     target: ts.ScriptTarget.ESNext
   };
 
+  private _sourceFileCache: Record<string, SourceFile> = {};
+
   private _resolveModule(specifier: string, referrer: string): SourceFile {
+    util.log("host._resolveModule", { specifier, referrer });
     // Handle built-in assets specially.
     if (specifier.startsWith(ASSETS)) {
       const moduleName = specifier.split("/").pop()!;
+      if (moduleName in this._sourceFileCache) {
+        return this._sourceFileCache[moduleName];
+      }
       const assetName = moduleName.includes(".")
         ? moduleName
         : `${moduleName}.d.ts`;
       assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
       const sourceCode = assetSourceCode[assetName];
-      return {
+      const sourceFile = {
         moduleName,
         filename: specifier,
         mediaType: msg.MediaType.TypeScript,
         sourceCode
       };
+      this._sourceFileCache[moduleName] = sourceFile;
+      return sourceFile;
+    }
+    const sourceFile = fetchSourceFile(specifier, referrer);
+    assert(sourceFile.moduleName != null);
+    const { moduleName } = sourceFile;
+    if (!(moduleName! in this._sourceFileCache)) {
+      this._sourceFileCache[moduleName!] = sourceFile;
     }
-    return fetchSourceFile(specifier, referrer);
+    return sourceFile;
   }
 
   /* Deno specific APIs */
@@ -279,7 +296,7 @@ class Host implements ts.CompilerHost {
    * options which were ignored, or `undefined`.
    */
   configure(path: string, configurationText: string): ConfigureResponse {
-    util.log("compile.configure", path);
+    util.log("host.configure", path);
     const { config, error } = ts.parseConfigFileTextToJson(
       path,
       configurationText
@@ -344,13 +361,17 @@ class Host implements ts.CompilerHost {
   ): ts.SourceFile | undefined {
     assert(!shouldCreateNewSourceFile);
     util.log("getSourceFile", fileName);
-    const SourceFile = this._resolveModule(fileName, ".");
-    if (!SourceFile || !SourceFile.sourceCode) {
+    const sourceFile =
+      fileName in this._sourceFileCache
+        ? this._sourceFileCache[fileName]
+        : this._resolveModule(fileName, ".");
+    assert(sourceFile != null);
+    if (!sourceFile.sourceCode) {
       return undefined;
     }
     return ts.createSourceFile(
       fileName,
-      SourceFile.sourceCode,
+      sourceFile.sourceCode,
       languageVersion
     );
   }
@@ -364,26 +385,37 @@ class Host implements ts.CompilerHost {
     containingFile: string
   ): Array<ts.ResolvedModuleFull | undefined> {
     util.log("resolveModuleNames()", { moduleNames, containingFile });
+    const typeDirectives: Record<string, string> | undefined =
+      containingFile in this._sourceFileCache
+        ? this._sourceFileCache[containingFile].typeDirectives
+        : undefined;
     return moduleNames.map(
       (moduleName): ts.ResolvedModuleFull | undefined => {
-        const SourceFile = this._resolveModule(moduleName, containingFile);
-        if (SourceFile.moduleName) {
-          const resolvedFileName = SourceFile.moduleName;
+        const mappedModuleName = getMappedModuleName(
+          moduleName,
+          containingFile,
+          typeDirectives
+        );
+        const sourceFile = this._resolveModule(
+          mappedModuleName,
+          containingFile
+        );
+        if (sourceFile.moduleName) {
+          const resolvedFileName = sourceFile.moduleName;
           // This flags to the compiler to not go looking to transpile functional
           // code, anything that is in `/$asset$/` is just library code
           const isExternalLibraryImport = moduleName.startsWith(ASSETS);
           const extension = getExtension(
             resolvedFileName,
-            SourceFile.mediaType
+            sourceFile.mediaType
           );
-          this.extensionCache[resolvedFileName] = extension;
+          this._extensionCache[resolvedFileName] = extension;
 
-          const r = {
+          return {
             resolvedFileName,
             isExternalLibraryImport,
             extension
           };
-          return r;
         } else {
           return undefined;
         }
@@ -409,7 +441,7 @@ class Host implements ts.CompilerHost {
       } else {
         assert(sourceFiles != null && sourceFiles.length == 1);
         const sourceFileName = sourceFiles![0].fileName;
-        const maybeExtension = this.extensionCache[sourceFileName];
+        const maybeExtension = this._extensionCache[sourceFileName];
 
         if (maybeExtension) {
           // NOTE: If it's a `.json` file we don't want to write it to disk.

js/type_directives.ts

@@ -0,0 +1,87 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+
+interface DirectiveInfo {
+  path: string;
+  start: number;
+  end: number;
+}
+
+/** Remap the module name based on any supplied type directives passed. */
+export function getMappedModuleName(
+  moduleName: string,
+  containingFile: string,
+  typeDirectives?: Record<string, string>
+): string {
+  if (containingFile.endsWith(".d.ts") && !moduleName.endsWith(".d.ts")) {
+    moduleName = `${moduleName}.d.ts`;
+  }
+  if (!typeDirectives) {
+    return moduleName;
+  }
+  if (moduleName in typeDirectives) {
+    return typeDirectives[moduleName];
+  }
+  return moduleName;
+}
+
+/** Matches directives that look something like this and parses out the value
+ * of the directive:
+ *
+ *      // @deno-types="./foo.d.ts"
+ * 
+ * [See Diagram](http://bit.ly/31nZPCF)
+ */
+const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi;
+
+/** Matches `import` or `export from` statements and parses out the value of the
+ * module specifier in the second capture group:
+ *
+ *      import * as foo from "./foo.js"
+ *      export { a, b, c } from "./bar.js"
+ *
+ * [See Diagram](http://bit.ly/2GSkJlF)
+ */
+const importExportRegEx = /(?:import|export)\s+[\s\S]*?from\s+(["'])((?:(?=(\\?))\3.)*?)\1/;
+
+/** Parses out any Deno type directives that are part of the source code, or
+ * returns `undefined` if there are not any.
+ */
+export function parseTypeDirectives(
+  sourceCode: string | undefined
+): Record<string, string> | undefined {
+  if (!sourceCode) {
+    return;
+  }
+
+  // collect all the directives in the file and their start and end positions
+  const directives: DirectiveInfo[] = [];
+  let maybeMatch: RegExpExecArray | null = null;
+  while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) {
+    const [matchString, , path] = maybeMatch;
+    const { index: start } = maybeMatch;
+    directives.push({
+      path,
+      start,
+      end: start + matchString.length
+    });
+  }
+  if (!directives.length) {
+    return;
+  }
+
+  // work from the last directive backwards for the next `import`/`export`
+  // statement
+  directives.reverse();
+  const directiveRecords: Record<string, string> = {};
+  for (const { path, start, end } of directives) {
+    const searchString = sourceCode.substring(end);
+    const maybeMatch = importExportRegEx.exec(searchString);
+    if (maybeMatch) {
+      const [, , fromPath] = maybeMatch;
+      directiveRecords[fromPath] = path;
+    }
+    sourceCode = sourceCode.substring(0, start);
+  }
+
+  return directiveRecords;
+}

tests/type_definitions/local/bar.d.ts

@@ -0,0 +1 @@
+export const bar: string;

tests/type_definitions/local/bar.js

@@ -0,0 +1 @@
+export const bar = "bar";

tests/type_definitions/local/foo.d.ts

@@ -0,0 +1,2 @@
+/** An exported value. */
+export const foo: string;

tests/type_definitions/local/foo.js

@@ -0,0 +1 @@
+export const foo = "foo";

tests/type_definitions/local/main.ts

@@ -0,0 +1,7 @@
+// @deno-types="./foo.d.ts"
+import * as foo from "./foo.js";
+
+// @deno-types="./bar.d.ts"
+import * as bar from "./bar.js";
+
+console.log(foo.foo, bar.bar);

tests/type_definitions/remote/main.ts

@@ -0,0 +1,10 @@
+// @deno-types="https://unpkg.com/dayjs@1.8.15/index.d.ts"
+import dayjs from "https://unpkg.com/dayjs@1.8.15/esm/index.js";
+
+const date = dayjs();
+console.log(date.second());
+
+// @deno-types="https://unpkg.com/@types/lodash@4.14.136/index.d.ts"
+import * as _ from "https://unpkg.com/lodash-es@4.17.15/lodash.js";
+
+console.log(_.add(1, 2));