Rewrite wasm-bindgen with updated interface types proposal (#1882)

This commit is a pretty large scale rewrite of the internals of wasm-bindgen. No user-facing changes are expected as a result of this PR, but due to the scale of changes here it's likely inevitable that at least something will break. I'm hoping to get more testing in though before landing!

The purpose of this PR is to update wasm-bindgen to the current state of the interface types proposal. The wasm-bindgen tool was last updated when it was still called "WebIDL bindings" so it's been awhile! All support is now based on https://github.com/bytecodealliance/wasm-interface-types which defines parsers/binary format/writers/etc for wasm-interface types.

This is a pretty massive PR and unfortunately can't really be split up any more afaik. I don't really expect realistic review of all the code here (or commits), but some high-level changes are:

* Interface types now consists of a set of "adapter functions". The IR in wasm-bindgen is modeled the same way not.
* Each adapter function has a list of instructions, and these instructions work at a higher level than wasm itself, for example with strings.
* The wasm-bindgen tool has a suite of instructions which are specific to it and not present in the standard. (like before with webidl bindings)
* The anyref/multi-value transformations are now greatly simplified. They're simply "optimization passes" over adapter functions, removing instructions that are otherwise present. This way we don't have to juggle so much all over the place, and instructions always have the same meaning.

azure-pipelines.yml

@@ -96,6 +96,10 @@ jobs:
         displayName: "wasm-bindgen-cli-support tests"
       - script: cargo test -p wasm-bindgen-cli
         displayName: "wasm-bindgen-cli tests"
+      - script: cargo test -p wasm-bindgen-anyref-xform
+        displayName: "wasm-bindgen-anyref-xform tests"
+      - script: cargo test -p wasm-bindgen-multi-value-xform
+        displayName: "wasm-bindgen-multi-value-xform tests"
 
   - job: test_web_sys
     displayName: "Run web-sys crate tests"

crates/anyref-xform/Cargo.toml

@@ -13,4 +13,14 @@ edition = '2018'
 
 [dependencies]
 anyhow = "1.0"
-walrus = "0.13.0"
+walrus = "0.14.0"
+
+[dev-dependencies]
+rayon = "1.0"
+wasmprinter = "0.2"
+wast = "3.0"
+wat = "1.0"
+
+[[test]]
+name = "all"
+harness = false

crates/anyref-xform/src/lib.rs

@@ -44,6 +44,12 @@ pub struct Context {
     table: Option<TableId>,
 }
 
+pub struct Meta {
+    pub table: TableId,
+    pub alloc: Option<FunctionId>,
+    pub drop_slice: Option<FunctionId>,
+}
+
 struct Transform<'a> {
     cx: &'a mut Context,
 
@@ -161,7 +167,7 @@ impl Context {
         })
     }
 
-    pub fn run(&mut self, module: &mut Module) -> Result<(), Error> {
+    pub fn run(&mut self, module: &mut Module) -> Result<Meta, Error> {
         let table = self.table.unwrap();
 
         // Inject a stack pointer global which will be used for managing the
@@ -171,6 +177,7 @@ impl Context {
 
         let mut heap_alloc = None;
         let mut heap_dealloc = None;
+        let mut drop_slice = None;
 
         // Find exports of some intrinsics which we only need for a runtime
         // implementation.
@@ -182,7 +189,8 @@ impl Context {
             match export.name.as_str() {
                 "__wbindgen_anyref_table_alloc" => heap_alloc = Some(f),
                 "__wbindgen_anyref_table_dealloc" => heap_dealloc = Some(f),
-                _ => {}
+                "__wbindgen_drop_anyref_slice" => drop_slice = Some(f),
+                _ => continue,
             }
         }
         let mut clone_ref = None;
@@ -226,7 +234,13 @@ impl Context {
             heap_dealloc,
             stack_pointer,
         }
-        .run(module)
+        .run(module)?;
+
+        Ok(Meta {
+            table,
+            alloc: heap_alloc,
+            drop_slice,
+        })
     }
 }
 
@@ -619,7 +633,7 @@ impl Transform<'_> {
         // with a fresh type we've been calculating so far. Give the function a
         // nice name for debugging and then we're good to go!
         let id = builder.finish(params, funcs);
-        let name = format!("{}_anyref_shim", name);
+        let name = format!("{} anyref shim", name);
         funcs.get_mut(id).name = Some(name);
         self.shims.insert(id);
         Ok((id, anyref_ty))

crates/anyref-xform/tests/all.rs

@@ -0,0 +1,258 @@
+//! A small test framework to execute a test function over all files in a
+//! directory.
+//!
+//! Each file in the directory has its own `CHECK-ALL` annotation indicating the
+//! expected output of the test. That can be automatically updated with
+//! `BLESS=1` in the environment. Otherwise the test are checked against the
+//! listed expectation.
+
+use anyhow::{anyhow, bail, Context, Result};
+use rayon::prelude::*;
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+use wast::parser::{Parse, Parser};
+
+fn main() {
+    run("tests".as_ref(), runtest);
+}
+
+fn runtest(test: &Test) -> Result<String> {
+    let wasm = wat::parse_file(&test.file)?;
+    let mut walrus = walrus::Module::from_buffer(&wasm)?;
+    let mut cx = wasm_bindgen_anyref_xform::Context::default();
+    cx.prepare(&mut walrus)?;
+    for directive in test.directives.iter() {
+        match &directive.kind {
+            DirectiveKind::Export(name) => {
+                let export = walrus
+                    .exports
+                    .iter()
+                    .find(|e| e.name == *name)
+                    .ok_or_else(|| anyhow!("failed to find export"))?;
+                cx.export_xform(export.id(), &directive.args, directive.ret_anyref);
+            }
+            DirectiveKind::Import(module, field) => {
+                let import = walrus
+                    .imports
+                    .iter()
+                    .find(|e| e.module == *module && e.name == *field)
+                    .ok_or_else(|| anyhow!("failed to find export"))?;
+                cx.import_xform(import.id(), &directive.args, directive.ret_anyref);
+            }
+            DirectiveKind::Table(idx) => {
+                cx.table_element_xform(*idx, &directive.args, directive.ret_anyref);
+            }
+        }
+    }
+    cx.run(&mut walrus)?;
+    walrus::passes::gc::run(&mut walrus);
+    let printed = wasmprinter::print_bytes(&walrus.emit_wasm())?;
+    Ok(printed)
+}
+
+fn run(dir: &Path, run: fn(&Test) -> Result<String>) {
+    let mut tests = Vec::new();
+    find_tests(dir, &mut tests);
+    let filter = std::env::args().nth(1);
+
+    let bless = env::var("BLESS").is_ok();
+    let tests = tests
+        .iter()
+        .filter(|test| {
+            if let Some(filter) = &filter {
+                if let Some(s) = test.file_name().and_then(|s| s.to_str()) {
+                    if !s.contains(filter) {
+                        return false;
+                    }
+                }
+            }
+            true
+        })
+        .collect::<Vec<_>>();
+
+    println!("\nrunning {} tests\n", tests.len());
+
+    let errors = tests
+        .par_iter()
+        .filter_map(|test| run_test(test, bless, run).err())
+        .collect::<Vec<_>>();
+
+    if !errors.is_empty() {
+        for msg in errors.iter() {
+            eprintln!("error: {:?}", msg);
+        }
+
+        panic!("{} tests failed", errors.len())
+    }
+
+    println!("test result: ok. {} passed\n", tests.len());
+}
+
+fn run_test(test: &Path, bless: bool, run: fn(&Test) -> anyhow::Result<String>) -> Result<()> {
+    (|| -> Result<_> {
+        let expected = Test::from_file(test)?;
+        let actual = run(&expected)?;
+        expected.check(&actual, bless)?;
+        Ok(())
+    })()
+    .context(format!("test failed - {}", test.display()))?;
+    Ok(())
+}
+
+fn find_tests(path: &Path, tests: &mut Vec<PathBuf>) {
+    for f in path.read_dir().unwrap() {
+        let f = f.unwrap();
+        if f.file_type().unwrap().is_dir() {
+            find_tests(&f.path(), tests);
+            continue;
+        }
+        match f.path().extension().and_then(|s| s.to_str()) {
+            Some("wat") => {}
+            _ => continue,
+        }
+        tests.push(f.path());
+    }
+}
+
+struct Test {
+    file: PathBuf,
+    directives: Vec<Directive>,
+    assertion: Option<String>,
+}
+
+struct Directive {
+    args: Vec<(usize, bool)>,
+    ret_anyref: bool,
+    kind: DirectiveKind,
+}
+
+enum DirectiveKind {
+    Import(String, String),
+    Export(String),
+    Table(u32),
+}
+
+impl Test {
+    fn from_file(path: &Path) -> Result<Test> {
+        let contents = fs::read_to_string(path)?;
+        let mut iter = contents.lines();
+        let mut assertion = None;
+        let mut directives = Vec::new();
+        while let Some(line) = iter.next() {
+            if line.starts_with("(; CHECK-ALL:") {
+                let mut pattern = String::new();
+                while let Some(line) = iter.next() {
+                    if line == ";)" {
+                        break;
+                    }
+                    pattern.push_str(line);
+                    pattern.push_str("\n");
+                }
+                while pattern.ends_with("\n") {
+                    pattern.pop();
+                }
+                if iter.next().is_some() {
+                    bail!("CHECK-ALL must be at the end of the file");
+                }
+                assertion = Some(pattern);
+                continue;
+            }
+
+            if !line.starts_with(";; @xform") {
+                continue;
+            }
+            let directive = &line[9..];
+            let buf = wast::parser::ParseBuffer::new(directive)?;
+            directives.push(wast::parser::parse::<Directive>(&buf)?);
+        }
+        Ok(Test {
+            file: path.to_path_buf(),
+            directives,
+            assertion,
+        })
+    }
+
+    fn check(&self, output: &str, bless: bool) -> Result<()> {
+        if bless {
+            update_output(&self.file, output)
+        } else if let Some(pattern) = &self.assertion {
+            if output == pattern {
+                return Ok(());
+            }
+            bail!(
+                "expected\n    {}\n\nactual\n    {}",
+                pattern.replace("\n", "\n    "),
+                output.replace("\n", "\n    ")
+            );
+        } else {
+            bail!(
+                "no test assertions were found in this file, but you can \
+                 rerun tests with `BLESS=1` to automatically add assertions \
+                 to this file"
+            );
+        }
+    }
+}
+
+fn update_output(path: &Path, output: &str) -> Result<()> {
+    let contents = fs::read_to_string(path)?;
+    let start = contents.find("(; CHECK-ALL:").unwrap_or(contents.len());
+
+    let mut new_output = String::new();
+    for line in output.lines() {
+        new_output.push_str(line);
+        new_output.push_str("\n");
+    }
+    let new = format!(
+        "{}\n\n(; CHECK-ALL:\n{}\n;)\n",
+        contents[..start].trim(),
+        new_output.trim_end()
+    );
+    fs::write(path, new)?;
+    Ok(())
+}
+
+impl<'a> Parse<'a> for Directive {
+    fn parse(parser: Parser<'a>) -> wast::parser::Result<Self> {
+        use wast::kw;
+        wast::custom_keyword!(anyref_owned);
+        wast::custom_keyword!(anyref_borrowed);
+        wast::custom_keyword!(other);
+
+        let kind = if parser.peek::<kw::import>() {
+            parser.parse::<kw::import>()?;
+            DirectiveKind::Import(parser.parse()?, parser.parse()?)
+        } else if parser.peek::<kw::export>() {
+            parser.parse::<kw::export>()?;
+            DirectiveKind::Export(parser.parse()?)
+        } else {
+            parser.parse::<kw::table>()?;
+            DirectiveKind::Table(parser.parse()?)
+        };
+        let mut args = Vec::new();
+        parser.parens(|p| {
+            let mut i = 0;
+            while !p.is_empty() {
+                if parser.peek::<anyref_owned>() {
+                    parser.parse::<anyref_owned>()?;
+                    args.push((i, true));
+                } else if parser.peek::<anyref_borrowed>() {
+                    parser.parse::<anyref_borrowed>()?;
+                    args.push((i, false));
+                } else {
+                    parser.parse::<other>()?;
+                }
+                i += 1;
+            }
+            Ok(())
+        })?;
+
+        let ret_anyref = parser.parse::<Option<anyref_owned>>()?.is_some();
+        Ok(Directive {
+            args,
+            ret_anyref,
+            kind,
+        })
+    }
+}

crates/anyref-xform/tests/anyref-param-owned.wat

@@ -0,0 +1,31 @@
+;; @xform export "foo" (anyref_owned)
+
+(module
+  (func $foo (export "foo") (param i32))
+  (func $alloc (export "__wbindgen_anyref_table_alloc") (result i32)
+    i32.const 0)
+  (func $dealloc (export "__wbindgen_anyref_table_dealloc") (param i32))
+)
+
+(; CHECK-ALL:
+(module
+  (type (;0;) (func (result i32)))
+  (type (;1;) (func (param i32)))
+  (type (;2;) (func (param anyref)))
+  (func $foo anyref shim (type 2) (param anyref)
+    (local i32)
+    call $alloc
+    local.tee 1
+    local.get 0
+    table.set 0
+    local.get 1
+    call $foo)
+  (func $alloc (type 0) (result i32)
+    i32.const 0)
+  (func $foo (type 1) (param i32))
+  (func $dealloc (type 1) (param i32))
+  (table (;0;) 32 anyref)
+  (export "foo" (func $foo anyref shim))
+  (export "__wbindgen_anyref_table_alloc" (func $alloc))
+  (export "__wbindgen_anyref_table_dealloc" (func $dealloc)))
+;)