Refactor snapshot build (#2825)

Instead of using core/snapshot_creator.rs, instead two crates are
introduced which allow building the snapshot during build.rs.

Rollup is removed and replaced with our own bundler. This removes
the Node build dependency. Modules in //js now use Deno-style imports
with file extensions, rather than Node style extensionless imports.

This improves incremental build time when changes are made to //js files
by about 40 seconds.

.appveyor.yml

@@ -200,10 +200,6 @@ test_script:
   - ps: Exec { & python tools\test.py --build-dir $env:DENO_BUILD_PATH }
 
 after_test:
-  # Delete the the rollup cache, which is unreliable, so that it doesn't get
-  # persisted in the appveyor cache.
-  - ps: if (Get-SaveCache) { Delete-Tree "$env:DENO_BUILD_PATH\.rpt2_cache" }
-
   # Stop sccache and show stats.
   - ps: sccache --stop-server
 
@@ -215,36 +211,15 @@ after_test:
         throw "Build should be up-to-date but isn't."
       }
 
-  # Verify that javascript and typescript files which are bundled by rollup are
-  # listed explicitly in cli\BUILD.gn. This is not an air-tight check.
-  # TODO: make rollup or another bundler write a depfile.
+  # Verify that the bundled javascript and typescript files are listed
+  # explicitly in cli_snapshots\BUILD.gn. This is not an air-tight check.
   - ps: |-
       $ignore = "test_util.ts", "unit_tests.ts", "unit_test_runner.ts", "*_test.ts"
       Get-ChildItem "js" -File -Force -Name                               |
         where   { $name = $_; -not ($ignore | where { $name -like $_ }) } |
-        where   { -not (Select-String -Pattern $_ -Path cli\BUILD.gn `
+        where   { -not (Select-String -Pattern $_ -Path cli_snapshots\BUILD.gn `
                                       -SimpleMatch -CaseSensitive) }      |
-        foreach { throw "$_ should be listed in cli\BUILD.gn but isn't." }
-
-  # Verify that generated ninja files do not use absolute path names.
-  # If they do, it makes ccache/sccache much less effective.
-  - ps: |-
-      $trap = "NO_ABS_PATH_PLS"
-      $dir = "$env:APPVEYOR_BUILD_FOLDER\out\$trap"
-      Exec { gn gen $dir | Out-Null }
-      $files = Get-Tree $dir -File -Force -Recurse | where Extension -ne ".dll"
-      Select-String $trap -Path $files -SimpleMatch | where {
-        # V8 took the liberty to produce an absolute path in their ninja
-        # output. We can't do much about that, so we just ignore it.
-        $_.Line -notmatch "v8/builtins-generated/bytecodes-builtins-list.h"
-      } | tee -Variable line_matches
-      if ($line_matches) {
-        $ctx = $line_matches.Line                                          |
-          Select-String "[^\s;,]*[\s=]*[^\s;,=]*$trap[^\s;,]*" -AllMatches |
-          foreach { $_.Matches.Value -replace '\$(.)', '$1' }              |
-          sort -Unique
-        throw @("Absolute path(s) found in build script:") + $ctx -join "`n  "
-      }
+        foreach { throw "$_ should be listed in cli_snapshots\BUILD.gn but isn't." }
 
   # If this build is going to be deployed, build a zip file.
   - ps: |-

Cargo.toml

@@ -3,4 +3,6 @@ members = [
   "cli",
   "core",
   "tools/hyper_hello",
+  "deno_typescript",
+  "cli_snapshots",
 ]

cli/BUILD.gn

@@ -1,16 +1,23 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import("//build/toolchain/cc_wrapper.gni")
 import("//build_extra/rust/rust.gni")
-import("//third_party/v8/gni/snapshot_toolchain.gni")
-import("//third_party/v8/gni/v8.gni")
-import("deno.gni")
 
 main_extern = [
   {
     label = "../core:deno"
     crate_name = "deno"
     crate_type = "rlib"
   },
+  {
+    label = "../cli_snapshots:deno_cli_snapshots"
+    crate_name = "deno_cli_snapshots"
+    crate_type = "rlib"
+  },
+  {
+    label = "../deno_typescript:deno_typescript"
+    crate_name = "deno_typescript"
+    crate_type = "rlib"
+  },
   {
     label = "$rust_build:serde_derive"
     crate_name = "serde_derive"
@@ -61,109 +68,6 @@ if (is_posix) {
   main_extern_rlib += [ "nix" ]
 }
 
-ts_sources = [
-  "../js/base64.ts",
-  "../js/blob.ts",
-  "../js/body.ts",
-  "../js/buffer.ts",
-  "../js/build.ts",
-  "../js/chmod.ts",
-  "../js/chown.ts",
-  "../js/colors.ts",
-  "../js/compiler.ts",
-  "../js/console.ts",
-  "../js/console_table.ts",
-  "../js/copy_file.ts",
-  "../js/core.ts",
-  "../js/custom_event.ts",
-  "../js/deno.ts",
-  "../js/diagnostics.ts",
-  "../js/dir.ts",
-  "../js/dispatch.ts",
-  "../js/dispatch_json.ts",
-  "../js/dispatch_minimal.ts",
-  "../js/dom_file.ts",
-  "../js/dom_types.ts",
-  "../js/dom_util.ts",
-  "../js/error_stack.ts",
-  "../js/errors.ts",
-  "../js/event.ts",
-  "../js/event_target.ts",
-  "../js/fetch.ts",
-  "../js/file_info.ts",
-  "../js/files.ts",
-  "../js/form_data.ts",
-  "../js/format_error.ts",
-  "../js/get_random_values.ts",
-  "../js/globals.ts",
-  "../js/headers.ts",
-  "../js/io.ts",
-  "../js/lib.deno_runtime.d.ts",
-  "../js/lib.web_assembly.d.ts",
-  "../js/link.ts",
-  "../js/location.ts",
-  "../js/main.ts",
-  "../js/make_temp_dir.ts",
-  "../js/metrics.ts",
-  "../js/mkdir.ts",
-  "../js/mock_builtin.js",
-  "../js/net.ts",
-  "../js/os.ts",
-  "../js/performance.ts",
-  "../js/permissions.ts",
-  "../js/plugins.d.ts",
-  "../js/process.ts",
-  "../js/read_dir.ts",
-  "../js/read_file.ts",
-  "../js/read_link.ts",
-  "../js/remove.ts",
-  "../js/rename.ts",
-  "../js/repl.ts",
-  "../js/request.ts",
-  "../js/resources.ts",
-  "../js/stat.ts",
-  "../js/symlink.ts",
-  "../js/text_encoding.ts",
-  "../js/timers.ts",
-  "../js/truncate.ts",
-  "../js/type_directives.ts",
-  "../js/types.ts",
-  "../js/url.ts",
-  "../js/url_search_params.ts",
-  "../js/util.ts",
-  "../js/utime.ts",
-  "../js/version.ts",
-  "../js/window.ts",
-  "../js/workers.ts",
-  "../js/write_file.ts",
-  "../js/xeval.ts",
-  "../tsconfig.json",
-
-  # Listing package.json and yarn.lock as sources ensures the bundle is rebuilt
-  # when npm packages are added/removed or their contents changes.
-  "../package.json",
-  "../third_party/yarn.lock",
-]
-
-# When Cargo is driving the build, GN/Ninja are used to produce these non-Rust
-# targets. Cargo handles all Rust source files and the final linking step.
-group("deno_deps") {
-  deps = [
-    ":snapshot_compiler",
-    ":snapshot_deno",
-  ]
-}
-
-# Optimized dependencies for cross compiled builds.
-# This can be removed once we get snapshots into cross compiled builds.
-group("deno_deps_cross") {
-  testonly = true
-  deps = [
-    ":compiler_bundle",
-    ":main_bundle",
-  ]
-}
-
 # Reads the cargo info from Cargo.toml
 deno_cargo_info = exec_script("../build_extra/rust/get_cargo_info.py",
                               [ rebase_path("Cargo.toml", root_build_dir) ],
@@ -173,9 +77,6 @@ rust_executable("deno") {
   source_root = "main.rs"
   extern = main_extern
   extern_rlib = main_extern_rlib
-  deps = [
-    ":deno_deps",
-  ]
 
   # Extract version from Cargo.toml
   # TODO integrate this into rust.gni by allowing the rust_executable template
@@ -190,41 +91,10 @@ rust_test("cli_test") {
   source_root = "main.rs"
   extern = main_extern
   extern_rlib = main_extern_rlib
-  deps = [
-    ":deno_deps",
-  ]
 
   # Extract version from Cargo.toml
   inputs = [
     "Cargo.toml",
   ]
   env = [ "CARGO_PKG_VERSION=${deno_cargo_info.version}" ]
 }
-
-bundle("main_bundle") {
-  sources = ts_sources
-  out_dir = "$target_gen_dir/bundle/"
-  out_name = "main"
-}
-
-bundle("compiler_bundle") {
-  sources = ts_sources
-  out_dir = "$target_gen_dir/bundle/"
-  out_name = "compiler"
-}
-
-# Generates $target_gen_dir/snapshot_deno.bin
-snapshot("snapshot_deno") {
-  source_root = "$target_gen_dir/bundle/main.js"
-  deps = [
-    ":main_bundle",
-  ]
-}
-
-# Generates $target_gen_dir/snapshot_compiler.bin
-snapshot("snapshot_compiler") {
-  source_root = "$target_gen_dir/bundle/compiler.js"
-  deps = [
-    ":compiler_bundle",
-  ]
-}

cli/Cargo.toml

@@ -52,6 +52,8 @@ tokio-rustls = "0.10.0"
 tokio-threadpool = "0.1.15"
 url = "1.7.2"
 utime = "0.2.1"
+deno_cli_snapshots = { path = "../cli_snapshots" }
+deno_typescript = { path = "../deno_typescript" }
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.7"

cli/assets.rs

@@ -1,59 +1,8 @@
 static DENO_RUNTIME: &str = include_str!("../js/lib.deno_runtime.d.ts");
 
-macro_rules! inc {
-  ($e:expr) => {
-    Some(include_str!(concat!(
-      "../third_party/node_modules/typescript/lib/",
-      $e
-    )))
-  };
-}
-
 pub fn get_source_code(name: &str) -> Option<&'static str> {
   match name {
     "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
-    "lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"),
-    "lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"),
-    "lib.es2015.d.ts" => inc!("lib.es2015.d.ts"),
-    "lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"),
-    "lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"),
-    "lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"),
-    "lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"),
-    "lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"),
-    "lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"),
-    "lib.es2015.symbol.wellknown.d.ts" => {
-      inc!("lib.es2015.symbol.wellknown.d.ts")
-    }
-    "lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"),
-    "lib.es2016.d.ts" => inc!("lib.es2016.d.ts"),
-    "lib.es2017.d.ts" => inc!("lib.es2017.d.ts"),
-    "lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"),
-    "lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"),
-    "lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"),
-    "lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"),
-    "lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"),
-    "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"),
-    "lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"),
-    "lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"),
-    "lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"),
-    "lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"),
-    "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"),
-    "lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"),
-    "lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"),
-    "lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"),
-    "lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"),
-    "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"),
-    "lib.es2020.string.d.ts" => inc!("lib.es2020.string.d.ts"),
-    "lib.es2020.symbol.wellknown.d.ts" => {
-      inc!("lib.es2020.symbol.wellknown.d.ts")
-    }
-    "lib.es5.d.ts" => inc!("lib.es5.d.ts"),
-    "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
-    "lib.esnext.array.d.ts" => inc!("lib.esnext.array.d.ts"),
-    "lib.esnext.asynciterable.d.ts" => inc!("lib.esnext.asynciterable.d.ts"),
-    "lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
-    "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
-    "lib.esnext.symbol.d.ts" => inc!("lib.esnext.symbol.d.ts"),
-    _ => None,
+    _ => deno_typescript::get_asset(name),
   }
 }