Implement support for async functions in Wasmtime

This is an implementation of [RFC 2] in Wasmtime which is to support
`async`-defined host functions. At a high level support is added by
executing WebAssembly code that might invoke an asynchronous host
function on a separate native stack. When the host function's future is
not ready we switch back to the main native stack to continue execution.

There's a whole bunch of details in this commit, and it's a bit much to
go over them all here in this commit message. The most important changes
here are:

* A new `wasmtime-fiber` crate has been written to manage the low-level
  details of stack-switching. Unixes use `mmap` to allocate a stack and
  Windows uses the native fibers implementation. We'll surely want to
  refactor this to move stack allocation elsewhere in the future. Fibers
  are intended to be relatively general with a lot of type paremters to
  fling values back and forth across suspension points. The whole crate
  is a giant wad of `unsafe` unfortunately and involves handwritten
  assembly with custom dwarf CFI directives to boot. Definitely deserves
  a close eye in review!

* The `Store` type has two new methods -- `block_on` and `on_fiber`
  which bridge between the async and non-async worlds. Lots of unsafe
  fiddly bits here as we're trying to communicate context pointers
  between disparate portions of the code. Extra eyes and care in review
  is greatly appreciated.

* The APIs for binding `async` functions are unfortunately pretty ugly
  in `Func`. This is mostly due to language limitations and compiler
  bugs (I believe) in Rust. Instead of `Func::wrap` we have a
  `Func::wrapN_async` family of methods, and we've also got a whole
  bunch of `Func::getN_async` methods now too. It may be worth
  rethinking the API of `Func` to try to make the documentation page
  actually grok'able.

This isn't super heavily tested but the various test should suffice for
engaging hopefully nearly all the infrastructure in one form or another.
This is just the start though!

[RFC 2]: bytecodealliance/rfcs#2

.github/workflows/main.yml

@@ -67,7 +67,7 @@ jobs:
     name: Doc - build the API documentation
     runs-on: ubuntu-latest
     env:
-      RUSTDOCFLAGS: -Dbroken_intra_doc_links
+      RUSTDOCFLAGS: -Dbroken_intra_doc_links --cfg nightlydoc
       OPENVINO_SKIP_LINKING: 1
     steps:
     - uses: actions/checkout@v2
@@ -120,6 +120,8 @@ jobs:
     - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features wat
     - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features lightbeam
     - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features jitdump
+    - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features cache
+    - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features async
 
     # Check some feature combinations of the `wasmtime-c-api` crate
     - run: cargo check --manifest-path crates/c-api/Cargo.toml --no-default-features

crates/fiber/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "wasmtime-fiber"
+version = "0.22.0"
+authors = ["The Wasmtime Project Developers"]
+description = "Fiber support for Wasmtime"
+license = "Apache-2.0 WITH LLVM-exception"
+repository = "https://github.com/bytecodealliance/wasmtime"
+edition = "2018"
+
+# We link to some native code with symbols that don't change often, so let Cargo
+# know that we can't show up multiple times in a crate graph. If this is an
+# issue in the future we should tweak the build script to set `#define`
+# directives or similar to embed a version number of this crate in symbols.
+links = "wasmtime-fiber-shims"
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2.80"
+
+[target.'cfg(windows)'.dependencies.winapi]
+version = "0.3.9"
+features = [
+  "fibersapi",
+  "winbase",
+]
+
+[build-dependencies]
+cc = "1.0"
+
+[dev-dependencies]
+backtrace = "0.3"

crates/fiber/build.rs

@@ -0,0 +1,23 @@
+use std::env;
+
+fn main() {
+    let mut build = cc::Build::new();
+    let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
+    let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap();
+    let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
+    if family == "windows" {
+        build.file("src/arch/windows.c");
+    } else if arch == "x86_64" {
+        build.file("src/arch/x86_64.S");
+    } else if arch == "aarch64" {
+        build.file("src/arch/aarch64.S");
+    } else {
+        panic!(
+            "wasmtime doesn't support fibers on platform: {}",
+            env::var("TARGET").unwrap()
+        );
+    }
+    build.define(&format!("CFG_TARGET_OS_{}", os), None);
+    build.define(&format!("CFG_TARGET_ARCH_{}", arch), None);
+    build.compile("wasmtime-fiber");
+}

crates/fiber/src/arch/aarch64.S

@@ -0,0 +1,111 @@
+// A WORD OF CAUTION
+//
+// This entire file basically needs to be kept in sync with itself. It's not
+// really possible to modify just one bit of this file without understanding
+// all the other bits. Documentation tries to reference various bits here and
+// there but try to make sure to read over everything before tweaking things!
+//
+// Also at this time this file is heavily based off the x86_64 file, so you'll
+// probably want to read that one as well.
+
+#define GLOBL(fnname) .globl fnname
+#define TYPE(fnname) .type fnname,@function
+#define FUNCTION(fnname) fnname
+#define SIZE(fnname) .size fnname,.-fnname
+
+// fn(top_of_stack(%x0): *mut u8)
+GLOBL(wasmtime_fiber_switch)
+.p2align 2
+TYPE(wasmtime_fiber_switch)
+FUNCTION(wasmtime_fiber_switch):
+    // Save all callee-saved registers on the stack since we're assuming
+    // they're clobbered as a result of the stack switch.
+    str lr, [sp, -16]!
+    stp x20, x19, [sp, -16]!
+    stp x22, x21, [sp, -16]!
+    stp x24, x23, [sp, -16]!
+    stp x26, x25, [sp, -16]!
+    stp x28, x27, [sp, -16]!
+
+    // Load our previously saved stack pointer to resume to, and save off our
+    // current stack pointer on where to come back to eventually.
+    ldr x8, [x0, -0x10]
+    mov x9, sp
+    str x9, [x0, -0x10]
+
+    // Switch to the new stack and restore all our callee-saved registers after
+    // the switch and return to our new stack.
+    mov sp, x8
+    ldp x28, x27, [sp], 16
+    ldp x26, x25, [sp], 16
+    ldp x24, x23, [sp], 16
+    ldp x22, x21, [sp], 16
+    ldp x20, x19, [sp], 16
+    ldr lr, [sp], 16
+    ret
+SIZE(wasmtime_fiber_switch)
+
+// fn(
+//    top_of_stack(%x0): *mut u8,
+//    entry_point(%x1): extern fn(*mut u8, *mut u8),
+//    entry_arg0(%x2): *mut u8,
+// )
+GLOBL(wasmtime_fiber_init)
+.p2align 2
+TYPE(wasmtime_fiber_init)
+FUNCTION(wasmtime_fiber_init):
+    adr x8, wasmtime_fiber_start
+    stp x0, x8, [x0, -0x28] // x0 => x19, x8 => lr
+    stp x2, x1, [x0, -0x38] // x1 => x20, x2 => x21
+
+    // `wasmtime_fiber_switch` has an 0x60 byte stack, and we add 0x10 more for
+    // the original reserved 16 bytes.
+    add x8, x0, -0x70
+    str x8, [x0, -0x10]
+    ret
+SIZE(wasmtime_fiber_init)
+
+.p2align 2
+TYPE(wasmtime_fiber_start)
+FUNCTION(wasmtime_fiber_start):
+.cfi_startproc simple
+
+    // See the x86_64 file for more commentary on what these CFI directives are
+    // doing. Like over there note that the relative offsets to registers here
+    // match the frame layout in `wasmtime_fiber_switch`.
+    .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \
+	4,	      /* the byte length of this expression */ \
+	0x6f,         /* DW_OP_reg31(%sp) */ \
+	0x06,         /* DW_OP_deref */ \
+	0x23, 0x60    /* DW_OP_plus_uconst 0x60 */
+
+    .cfi_rel_offset lr, -0x10
+    .cfi_rel_offset x19, -0x18
+    .cfi_rel_offset x20, -0x20
+    .cfi_rel_offset x21, -0x28
+    .cfi_rel_offset x22, -0x30
+    .cfi_rel_offset x23, -0x38
+    .cfi_rel_offset x24, -0x40
+    .cfi_rel_offset x25, -0x48
+    .cfi_rel_offset x26, -0x50
+    .cfi_rel_offset x27, -0x58
+    .cfi_rel_offset x29, -0x60
+
+    // Load our two arguments from the stack, where x1 is our start procedure
+    // and x0 is its first argument. This also blows away the stack space used
+    // by those two arguments.
+    mov x0, x21
+    mov x1, x19
+
+    // ... and then we call the function! Note that this is a function call so
+    // our frame stays on the stack to backtrace through.
+    blr x20
+    // .. technically we shouldn't get here, and I would like to write in an
+    // instruction which just aborts, but I don't know such an instruction in
+    // aarch64 lan, and I would like to write in an instruction which just
+    // aborts, but I don't know such an instruction in aarch64 land
+    .cfi_endproc
+SIZE(wasmtime_fiber_start)
+
+.section .note.GNU-stack,"",%progbits
+

crates/fiber/src/arch/windows.c

@@ -0,0 +1,5 @@
+#include <windows.h>
+
+LPVOID wasmtime_fiber_get_current() {
+   return GetCurrentFiber();
+}

crates/fiber/src/arch/x86_64.S

@@ -0,0 +1,177 @@
+// A WORD OF CAUTION
+//
+// This entire file basically needs to be kept in sync with itself. It's not
+// really possible to modify just one bit of this file without understanding
+// all the other bits. Documentation tries to reference various bits here and
+// there but try to make sure to read over everything before tweaking things!
+
+.text
+
+#if CFG_TARGET_OS_macos
+
+#define GLOBL(fnname) .globl _##fnname
+#define TYPE(fnname)
+#define FUNCTION(fnname) _##fnname
+#define SIZE(fnname)
+
+#else
+
+#define GLOBL(fnname) .globl fnname
+#define TYPE(fnname) .type fnname,@function
+#define FUNCTION(fnname) fnname
+#define SIZE(fnname) .size fnname,.-fnname
+
+#endif
+
+// fn(top_of_stack(%rdi): *mut u8)
+GLOBL(wasmtime_fiber_switch)
+.align 16
+TYPE(wasmtime_fiber_switch)
+FUNCTION(wasmtime_fiber_switch):
+    // We're switching to arbitrary code somewhere else, so pessimistically
+    // assume that all callee-save register are clobbered. This means we need
+    // to save/restore all of them.
+    //
+    // Note that this order for saving is important since we use CFI directives
+    // below to point to where all the saved registers are.
+    pushq %rbp
+    pushq %rbx
+    pushq %r12
+    pushq %r13
+    pushq %r14
+    pushq %r15
+
+    // Load pointer that we're going to resume at and store where we're going
+    // to get resumed from. This is in accordance with the diagram at the top
+    // of unix.rs.
+    movq -0x10(%rdi), %rax
+    mov %rsp, -0x10(%rdi)
+
+    // Swap stacks and restore all our callee-saved registers
+    mov %rax, %rsp
+    popq %r15
+    popq %r14
+    popq %r13
+    popq %r12
+    popq %rbx
+    popq %rbp
+    ret
+SIZE(wasmtime_fiber_switch)
+
+// fn(
+//    top_of_stack(%rdi): *mut u8,
+//    entry_point(%rsi): extern fn(*mut u8, *mut u8),
+//    entry_arg0(%rdx): *mut u8,
+// )
+GLOBL(wasmtime_fiber_init)
+.align 16
+TYPE(wasmtime_fiber_init)
+FUNCTION(wasmtime_fiber_init):
+    // The first 16 bytes of the stack are reserved (see unix.rs) so we store
+    // the initial data used in `wasmtime_fiber_start` just below
+    // that.
+    movq %rdi, -0x18(%rdi)
+    movq %rsi, -0x20(%rdi)
+    movq %rdx, -0x28(%rdi)
+
+    // After these arguments is the return address of where to switch to,
+    // which for the first run is `wasmtime_fiber_start`.
+    lea FUNCTION(wasmtime_fiber_start)(%rip), %rax
+    movq %rax, -0x30(%rdi)
+
+    // And then we specify the stack pointer resumption should begin at. Our
+    // `wasmtime_fiber_switch` function saves 6 registers so we need to ensure
+    // that there's space for that as well. 0x30 + 6 * 8 == 0x60 here.
+    lea -0x60(%rdi), %rax
+    movq %rax, -0x10(%rdi)
+    ret
+SIZE(wasmtime_fiber_init)
+
+// This is a pretty special function that has no real signature. Its use is to
+// be the "base" function of all fibers. This entrypoint is used in
+// `wasmtime_fiber_init` to bootstrap the execution of a new fiber.
+//
+// We also use this function as a persistent frame on the stack to emit dwarf
+// information to unwind into the caller. This allows us to unwind from the
+// fiber's stack back to the main stack that the fiber was called from. We use
+// special dwarf directives here to do so since this is a pretty nonstandard
+// function.
+//
+// If you're curious a decent introduction to CFI things and unwinding is at
+// https://www.imperialviolet.org/2017/01/18/cfi.html
+.align 16
+TYPE(wasmtime_fiber_start)
+FUNCTION(wasmtime_fiber_start):
+// Use the `simple` directive on the startproc here which indicates that some
+// default settings for the platform are omitted, since this function is so
+// nonstandard
+.cfi_startproc simple
+    // This is where things get special, we're specifying a custom dwarf
+    // expression for how to calculate the CFA. The goal here is that we need
+    // to load the parent's stack pointer just before the call it made into
+    // `wasmtime_fiber_switch`. Note that the CFA value changes over time as
+    // well because a fiber may be resumed multiple times from different points
+    // on the original stack. This means that our custom CFA directive involves
+    // `DW_OP_deref`, which loads data from memory.
+    //
+    // The expression we're encoding here is that the CFA, the stack pointer of
+    // whatever called into `wasmtime_fiber_start`, is:
+    //
+    //        *($rsp + 0x18) + 0x38
+    //
+    // $rsp is the stack pointer of `wasmtime_fiber_start` at the time the next
+    // instruction after the `.cfi_escape` is executed. Our $rsp at the start
+    // of this function is 3 words below stack start (0xAff0 in
+    // the diagram in unix.rs). The $rsp to resume at is at 0xAff0, so we
+    // add an offset to $rsp to get to that memory location and then we
+    // dereference it.
+    //
+    // After dereferencing, though, we have the $rsp value for
+    // `wasmtime_fiber_switch` itself. That's a weird function which sort of
+    // and sort of doesn't exist on the stack.  We want to point to the caller
+    // of `wasmtime_fiber_switch`, so to do that we need to skip the stack space
+    // reserved by `wasmtime_fiber_switch`, which is the 6 saved registers plus
+    // the return address of the caller's `call` instruction. Hence we offset
+    // another 0x38 bytes.
+    .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \
+	5,            /* the byte length of this expression */ \
+	0x77, 0x18,   /* DW_OP_breg7 (%rsp) + 0x18 */ \
+	0x06,         /* DW_OP_deref */ \
+	0x23, 0x38    /* DW_OP_plus_uconst 0x38 */
+
+    // And now after we've indicated where our CFA is for our parent function,
+    // we can define that where all of the saved registers are located. This
+    // uses standard `.cfi` directives which indicate that these registers are
+    // all stored relative to the CFA. Note that this order is kept in sync
+    // with the above register spills in `wasmtime_fiber_switch`.
+    .cfi_rel_offset rip, -8
+    .cfi_rel_offset rbp, -16
+    .cfi_rel_offset rbx, -24
+    .cfi_rel_offset r12, -32
+    .cfi_rel_offset r13, -40
+    .cfi_rel_offset r14, -48
+    .cfi_rel_offset r15, -56
+
+    // Update the CFA expression after each adjustment of $rsp as we load
+    // registers to call the entrypoint. The major change is that the $rsp
+    // offset is decreasing by 8, and for the last adjustment a 0 offset means
+    // we can use DW_OP_reg7.
+    popq %rdi
+    .cfi_escape 0x0f, 5, 0x77, 0x10, 0x06, 0x23, 0x38
+    popq %rax
+    .cfi_escape 0x0f, 5, 0x77, 0x08, 0x06, 0x23, 0x38
+    popq %rsi
+    .cfi_escape 0x0f, 4, 0x57,       0x06, 0x23, 0x38
+
+    // And finally head off into the fiber. Note the `callq` keeps this frame
+    // on the stack so all our CFI directives can be read. Additionally this
+    // is not expected to ever return, but for safety we put a `ud2` at the end.
+    callq *%rax
+    ud2
+    .cfi_endproc
+SIZE(wasmtime_fiber_start)
+
+// Mark that we don't need executable stack.
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif