refactor: snapshotting (#3753)

cli/Cargo.toml

@@ -19,6 +19,7 @@ name = "deno"
 path = "main.rs"
 
 [build-dependencies]
+deno_core = { path = "../core", version = "0.30.1" }
 deno_typescript = { path = "../deno_typescript", version = "0.30.1" }
 
 [dependencies]

cli/build.rs

@@ -1,7 +1,38 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+use deno_core::CoreOp;
+use deno_core::Isolate;
+use deno_core::Op;
+use deno_core::PinnedBuf;
+use deno_core::StartupData;
+use std::collections::HashMap;
 use std::env;
 use std::path::PathBuf;
 
+fn op_fetch_asset(
+  custom_assets: HashMap<String, PathBuf>,
+) -> impl Fn(&[u8], Option<PinnedBuf>) -> CoreOp {
+  move |control: &[u8], zero_copy_buf: Option<PinnedBuf>| -> CoreOp {
+    assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op.
+    let custom_assets = custom_assets.clone();
+    let name = std::str::from_utf8(control).unwrap();
+
+    let asset_code = if let Some(source_code) = deno_typescript::get_asset(name)
+    {
+      source_code.to_string()
+    } else if let Some(asset_path) = custom_assets.get(name) {
+      let source_code_vec =
+        std::fs::read(&asset_path).expect("Asset not found");
+      let source_code = std::str::from_utf8(&source_code_vec).unwrap();
+      source_code.to_string()
+    } else {
+      panic!("op_fetch_asset bad asset {}", name)
+    };
+
+    let vec = asset_code.into_bytes();
+    Op::Sync(vec.into_boxed_slice())
+  }
+}
+
 fn main() {
   // To debug snapshot issues uncomment:
   // deno_typescript::trace_serializer();
@@ -14,29 +45,49 @@ fn main() {
   let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
   let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
 
-  let custom_libs = vec![(
-    "lib.deno_runtime.d.ts".to_string(),
-    c.join("js/lib.deno_runtime.d.ts"),
-  )];
-
+  // Main snapshot
   let root_names = vec![c.join("js/main.ts")];
-  let bundle = o.join("CLI_SNAPSHOT.js");
-  let state =
-    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
-      .unwrap();
-  assert!(bundle.exists());
-  deno_typescript::mksnapshot_bundle(&bundle, state).unwrap();
-
-  let custom_libs = vec![(
+  let bundle_path = o.join("CLI_SNAPSHOT.js");
+  let snapshot_path = o.join("CLI_SNAPSHOT.bin");
+
+  let main_module_name =
+    deno_typescript::compile_bundle(&bundle_path, root_names)
+      .expect("Bundle compilation failed");
+  assert!(bundle_path.exists());
+
+  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
+
+  deno_typescript::mksnapshot_bundle(
+    runtime_isolate,
+    &snapshot_path,
+    &bundle_path,
+    &main_module_name,
+  )
+  .expect("Failed to create snapshot");
+
+  // Compiler snapshot
+  let root_names = vec![c.join("js/compiler.ts")];
+  let bundle_path = o.join("COMPILER_SNAPSHOT.js");
+  let snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
+  let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
+  custom_libs.insert(
     "lib.deno_runtime.d.ts".to_string(),
     c.join("js/lib.deno_runtime.d.ts"),
-  )];
+  );
 
-  let root_names = vec![c.join("js/compiler.ts")];
-  let bundle = o.join("COMPILER_SNAPSHOT.js");
-  let state =
-    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
-      .unwrap();
-  assert!(bundle.exists());
-  deno_typescript::mksnapshot_bundle_ts(&bundle, state).unwrap();
+  let main_module_name =
+    deno_typescript::compile_bundle(&bundle_path, root_names)
+      .expect("Bundle compilation failed");
+  assert!(bundle_path.exists());
+
+  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
+  runtime_isolate.register_op("fetch_asset", op_fetch_asset(custom_libs));
+
+  deno_typescript::mksnapshot_bundle_ts(
+    runtime_isolate,
+    &snapshot_path,
+    &bundle_path,
+    &main_module_name,
+  )
+  .expect("Failed to create snapshot");
 }

cli/js/compiler_bootstrap.ts

@@ -3,7 +3,7 @@
 import { ASSETS, Host } from "./compiler_host.ts";
 import { core } from "./core.ts";
 import * as dispatch from "./dispatch.ts";
-import { sendSync } from "./dispatch_json.ts";
+import { getAsset } from "./compiler_util.ts";
 
 // This registers ops that are available during the snapshotting process.
 const ops = core.ops();
@@ -26,9 +26,9 @@ export const oldProgram = ts.createProgram({
   host
 });
 
-/** A module loader which is concatenated into bundle files.  We read all static
- * assets during the snapshotting process, which is why this is located in
- * compiler_bootstrap. */
-export const bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, {
-  name: "bundle_loader.js"
-});
+/** A module loader which is concatenated into bundle files.
+ *
+ * We read all static assets during the snapshotting process, which is
+ * why this is located in compiler_bootstrap.
+ **/
+export const bundleLoader = getAsset("bundle_loader.js");

cli/js/compiler_host.ts

@@ -1,10 +1,8 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
-import { OUT_DIR, WriteFileCallback } from "./compiler_util.ts";
+import { OUT_DIR, WriteFileCallback, getAsset } from "./compiler_util.ts";
 import { cwd } from "./dir.ts";
-import { sendSync } from "./dispatch_json.ts";
-import * as dispatch from "./dispatch.ts";
 import { assert, notImplemented } from "./util.ts";
 import * as util from "./util.ts";
 
@@ -135,7 +133,7 @@ export class Host implements ts.CompilerHost {
       return sourceFile;
     }
     const name = url.includes(".") ? url : `${url}.d.ts`;
-    const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name });
+    const sourceCode = getAsset(name);
     return new SourceFile({
       url,
       filename,

cli/js/compiler_util.ts

@@ -7,7 +7,8 @@ import { ConfigureResponse, Host } from "./compiler_host.ts";
 import { SourceFile } from "./compiler_sourcefile.ts";
 import { sendSync } from "./dispatch_json.ts";
 import * as dispatch from "./dispatch.ts";
-import { TextEncoder } from "./text_encoding.ts";
+import { TextDecoder, TextEncoder } from "./text_encoding.ts";
+import { core } from "./core.ts";
 import * as util from "./util.ts";
 import { assert } from "./util.ts";
 import { writeFileSync } from "./write_file.ts";
@@ -89,12 +90,27 @@ function cache(
     assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
   }
 }
-
-const encoder = new TextEncoder();
+/**
+ * This op is called only during snapshotting.
+ *
+ * We really don't want to depend on JSON dispatch
+ * during snapshotting, so this op exchanges strings with Rust
+ * as raw byte arrays.
+ */
+export function getAsset(name: string): string {
+  const encoder = new TextEncoder();
+  const decoder = new TextDecoder();
+  const sourceCodeBytes = core.dispatch(
+    dispatch.OP_FETCH_ASSET,
+    encoder.encode(name)
+  );
+  return decoder.decode(sourceCodeBytes!);
+}
 
 /** Generates a `writeFile` function which can be passed to the compiler `Host`
  * to use when emitting files. */
 export function createWriteFile(state: WriteFileState): WriteFileCallback {
+  const encoder = new TextEncoder();
   if (state.type === CompilerRequestType.Compile) {
     return function writeFile(
       fileName: string,