diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9aab08a996b..1c2c989b49ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -356,11 +356,12 @@ jobs: -p wasmtime --no-default-features --features profiling -p wasmtime --no-default-features --features cache -p wasmtime --no-default-features --features async + -p wasmtime --no-default-features --features std -p wasmtime --no-default-features --features pooling-allocator -p wasmtime --no-default-features --features cranelift -p wasmtime --no-default-features --features component-model -p wasmtime --no-default-features --features runtime,component-model - -p wasmtime --no-default-features --features cranelift,wat,async,cache + -p wasmtime --no-default-features --features cranelift,wat,async,std,cache -p wasmtime --no-default-features --features winch -p wasmtime --no-default-features --features wmemcheck -p wasmtime --no-default-features --features wmemcheck,cranelift,runtime @@ -384,6 +385,12 @@ jobs: -p wasmtime --features incremental-cache -p wasmtime --all-features + - name: wasmtime-fiber + checks: | + -p wasmtime-fiber --no-default-features + -p wasmtime-fiber --no-default-features --features std + -p wasmtime-fiber --all-features + - name: wasmtime-cli checks: | -p wasmtime-cli --no-default-features @@ -432,6 +439,18 @@ jobs: env: GH_TOKEN: ${{ github.token }} + fiber_tests: + name: wasmtime-fiber tests + runs-on: ubuntu-latest + env: + CARGO_NDK_VERSION: 2.12.2 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/actions/install-rust + - run: cargo test -p wasmtime-fiber --no-default-features + # Checks for no_std support, ensure that crates can build on a no_std target no_std_checks: name: no_std checks diff --git a/crates/asm-macros/src/lib.rs b/crates/asm-macros/src/lib.rs index 0fd62e8b382f..92b04380b489 100644 --- a/crates/asm-macros/src/lib.rs +++ b/crates/asm-macros/src/lib.rs @@ -13,7 +13,7 @@ cfg_if::cfg_if! { #[macro_export] macro_rules! asm_func { ($name:expr, $body:expr $(, $($args:tt)*)?) => { - std::arch::global_asm!( + core::arch::global_asm!( concat!( ".p2align 4\n", ".private_extern _", $name, "\n", @@ -29,7 +29,7 @@ cfg_if::cfg_if! { #[macro_export] macro_rules! asm_func { ($name:expr, $body:expr $(, $($args:tt)*)?) => { - std::arch::global_asm!( + core::arch::global_asm!( concat!( ".def ", $name, "\n", ".scl 2\n", @@ -65,7 +65,7 @@ cfg_if::cfg_if! { #[macro_export] macro_rules! asm_func { ($name:expr, $body:expr $(, $($args:tt)*)?) => { - std::arch::global_asm!( + core::arch::global_asm!( concat!( ".p2align 4\n", ".hidden ", $name, "\n", diff --git a/crates/fiber/Cargo.toml b/crates/fiber/Cargo.toml index 8d83fd03553f..55f0087767ea 100644 --- a/crates/fiber/Cargo.toml +++ b/crates/fiber/Cargo.toml @@ -15,10 +15,10 @@ workspace = true anyhow = { workspace = true } cfg-if = { workspace = true } wasmtime-versioned-export-macros = { workspace = true } +wasmtime-asm-macros = { workspace = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true, features = ["mm", "param"] } -wasmtime-asm-macros = { workspace = true } [target.'cfg(windows)'.dependencies.windows-sys] workspace = true @@ -33,3 +33,9 @@ wasmtime-versioned-export-macros = { workspace = true } [dev-dependencies] backtrace = "0.3.68" + +[features] + +# Assume presence of the standard library. Allows propagating +# panic-unwinds across fiber invocations. +std = [] diff --git a/crates/fiber/build.rs b/crates/fiber/build.rs index 19e7a516259c..d14343fc0893 100644 --- a/crates/fiber/build.rs +++ b/crates/fiber/build.rs @@ -23,8 +23,8 @@ fn main() { build.file("src/windows.c"); build.define("VERSIONED_SUFFIX", Some(versioned_suffix!())); } else if arch == "s390x" { - println!("cargo:rerun-if-changed=src/unix/s390x.S"); - build.file("src/unix/s390x.S"); + println!("cargo:rerun-if-changed=src/stackswitch/s390x.S"); + build.file("src/stackswitch/s390x.S"); build.define("VERSIONED_SUFFIX", Some(versioned_suffix!())); } else { // assume that this is included via inline assembly in the crate itself, diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rs index 06a188a62330..11cbe912f11b 100644 --- a/crates/fiber/src/lib.rs +++ b/crates/fiber/src/lib.rs @@ -1,15 +1,22 @@ #![expect(clippy::allow_attributes, reason = "crate not migrated yet")] +#![no_std] +#[cfg(any(feature = "std", unix, windows))] +#[macro_use] +extern crate std; +extern crate alloc; + +use alloc::boxed::Box; use anyhow::Error; -use std::any::Any; -use std::cell::Cell; -use std::io; -use std::marker::PhantomData; -use std::ops::Range; -use std::panic::{self, AssertUnwindSafe}; +use core::cell::Cell; +use core::marker::PhantomData; +use core::ops::Range; cfg_if::cfg_if! { - if #[cfg(windows)] { + if #[cfg(not(feature = "std"))] { + mod nostd; + use nostd as imp; + } else if #[cfg(windows)] { mod windows; use windows as imp; } else if #[cfg(unix)] { @@ -20,6 +27,11 @@ cfg_if::cfg_if! { } } +// Our own stack switcher routines are used on Unix and no_std +// platforms, but not on Windows (it has its own fiber API). +#[cfg(any(unix, not(feature = "std")))] +pub(crate) mod stackswitch; + /// Represents an execution stack to use for a fiber. pub struct FiberStack(imp::FiberStack); @@ -31,14 +43,16 @@ fn _assert_send_sync() { _assert_sync::(); } +pub type Result = core::result::Result; + impl FiberStack { /// Creates a new fiber stack of the given size. - pub fn new(size: usize) -> io::Result { + pub fn new(size: usize) -> Result { Ok(Self(imp::FiberStack::new(size)?)) } /// Creates a new fiber stack of the given size. - pub fn from_custom(custom: Box) -> io::Result { + pub fn from_custom(custom: Box) -> Result { Ok(Self(imp::FiberStack::from_custom(custom)?)) } @@ -55,11 +69,7 @@ impl FiberStack { /// /// The caller must properly allocate the stack space with a guard page and /// make the pages accessible for correct behavior. - pub unsafe fn from_raw_parts( - bottom: *mut u8, - guard_size: usize, - len: usize, - ) -> io::Result { + pub unsafe fn from_raw_parts(bottom: *mut u8, guard_size: usize, len: usize) -> Result { Ok(Self(imp::FiberStack::from_raw_parts( bottom, guard_size, len, )?)) @@ -128,7 +138,8 @@ enum RunResult { Resuming(Resume), Yield(Yield), Returned(Return), - Panicked(Box), + #[cfg(feature = "std")] + Panicked(Box), } impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { @@ -140,7 +151,7 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { pub fn new( stack: FiberStack, func: impl FnOnce(Resume, &mut Suspend) -> Return + 'a, - ) -> io::Result { + ) -> Result { let inner = imp::Fiber::new(&stack.0, func)?; Ok(Self { @@ -177,7 +188,11 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { Err(y) } RunResult::Returned(r) => Ok(r), - RunResult::Panicked(payload) => std::panic::resume_unwind(payload), + #[cfg(feature = "std")] + RunResult::Panicked(_payload) => { + use std::panic; + panic::resume_unwind(_payload); + } } } @@ -222,11 +237,27 @@ impl Suspend { inner, _phantom: PhantomData, }; - let result = panic::catch_unwind(AssertUnwindSafe(|| (func)(initial, &mut suspend))); - suspend.inner.switch::(match result { - Ok(result) => RunResult::Returned(result), - Err(panic) => RunResult::Panicked(panic), - }); + + #[cfg(feature = "std")] + { + use std::panic::{self, AssertUnwindSafe}; + let result = panic::catch_unwind(AssertUnwindSafe(|| (func)(initial, &mut suspend))); + suspend.inner.switch::(match result { + Ok(result) => RunResult::Returned(result), + Err(panic) => RunResult::Panicked(panic), + }); + } + // Note that it is sound to omit the `catch_unwind` here: it + // will not result in unwinding going off the top of the fiber + // stack, because the code on the fiber stack is invoked via + // an extern "C" boundary which will panic on unwinds. + #[cfg(not(feature = "std"))] + { + let result = (func)(initial, &mut suspend); + suspend + .inner + .switch::(RunResult::Returned(result)); + } } } @@ -236,11 +267,11 @@ impl Drop for Fiber<'_, A, B, C> { } } -#[cfg(test)] +#[cfg(all(test))] mod tests { use super::{Fiber, FiberStack}; + use alloc::string::ToString; use std::cell::Cell; - use std::panic::{self, AssertUnwindSafe}; use std::rc::Rc; #[test] @@ -332,7 +363,10 @@ mod tests { } #[test] + #[cfg(feature = "std")] fn panics_propagated() { + use std::panic::{self, AssertUnwindSafe}; + let a = Rc::new(Cell::new(false)); let b = SetOnDrop(a.clone()); let fiber = diff --git a/crates/fiber/src/nostd.rs b/crates/fiber/src/nostd.rs new file mode 100644 index 000000000000..48bf3b6eae69 --- /dev/null +++ b/crates/fiber/src/nostd.rs @@ -0,0 +1,178 @@ +//! no_std implementation of fibers. +//! +//! This is a very stripped-down version of the Unix platform support, +//! but without mmap or guard pages, because on no_std systems we do +//! not assume that virtual memory exists. +//! +//! The stack layout is nevertheless the same (modulo the guard page) +//! as on Unix because we share its low-level implementations: +//! +//! ```text +//! 0xB000 +-----------------------+ <- top of stack +//! | &Cell | <- where to store results +//! 0xAff8 +-----------------------+ +//! | *const u8 | <- last sp to resume from +//! 0xAff0 +-----------------------+ <- 16-byte aligned +//! | | +//! ~ ... ~ <- actual native stack space to use +//! | | +//! 0x0000 +-----------------------+ +//! ``` +//! +//! Here `0xAff8` is filled in temporarily while `resume` is running. The fiber +//! started with 0xB000 as a parameter so it knows how to find this. +//! Additionally `resumes` stores state at 0xAff0 to restart execution, and +//! `suspend`, which has 0xB000 so it can find this, will read that and write +//! its own resumption information into this slot as well. + +use crate::stackswitch::*; +use crate::{Result, RunResult, RuntimeFiberStack}; +use alloc::boxed::Box; +use alloc::{vec, vec::Vec}; +use core::cell::Cell; +use core::ops::Range; + +// The no_std implementation is infallible in practice, but we use +// `anyhow::Error` here absent any better alternative. +pub type Error = anyhow::Error; + +pub struct FiberStack { + base: BasePtr, + len: usize, + /// Backing storage, if owned. Allocated once at startup and then + /// not reallocated afterward. + storage: Vec, +} + +struct BasePtr(*mut u8); + +unsafe impl Send for BasePtr {} +unsafe impl Sync for BasePtr {} + +const STACK_ALIGN: usize = 16; + +/// Align a pointer by incrementing it up to `align - 1` +/// bytes. `align` must be a power of two. Also updates the length as +/// appropriate so that `ptr + len` points to the same endpoint. +fn align_ptr(ptr: *mut u8, len: usize, align: usize) -> (*mut u8, usize) { + let ptr = ptr as usize; + let aligned = (ptr + align - 1) & !(align - 1); + let new_len = len - (aligned - ptr); + (aligned as *mut u8, new_len) +} + +impl FiberStack { + pub fn new(size: usize) -> Result { + // Round up the size to at least one page. + let size = core::cmp::max(4096, size); + let mut storage = vec![0; size]; + let (base, len) = align_ptr(storage.as_mut_ptr(), size, STACK_ALIGN); + Ok(FiberStack { + storage, + base: BasePtr(base), + len, + }) + } + + pub unsafe fn from_raw_parts(base: *mut u8, guard_size: usize, len: usize) -> Result { + Ok(FiberStack { + storage: vec![], + base: BasePtr(base.offset(isize::try_from(guard_size).unwrap())), + len, + }) + } + + pub fn is_from_raw_parts(&self) -> bool { + self.storage.is_empty() + } + + pub fn from_custom(_custom: Box) -> Result { + unimplemented!("Custom fiber stacks not supported in no_std fiber library") + } + + pub fn top(&self) -> Option<*mut u8> { + Some(self.base.0.wrapping_byte_add(self.len)) + } + + pub fn range(&self) -> Option> { + let base = self.base.0 as usize; + Some(base..base + self.len) + } + + pub fn guard_range(&self) -> Option> { + None + } +} + +pub struct Fiber; + +pub struct Suspend { + top_of_stack: *mut u8, +} + +extern "C" fn fiber_start(arg0: *mut u8, top_of_stack: *mut u8) +where + F: FnOnce(A, &mut super::Suspend) -> C, +{ + unsafe { + let inner = Suspend { top_of_stack }; + let initial = inner.take_resume::(); + super::Suspend::::execute(inner, initial, Box::from_raw(arg0.cast::())) + } +} + +impl Fiber { + pub fn new(stack: &FiberStack, func: F) -> Result + where + F: FnOnce(A, &mut super::Suspend) -> C, + { + unsafe { + let data = Box::into_raw(Box::new(func)).cast(); + wasmtime_fiber_init(stack.top().unwrap(), fiber_start::, data); + } + + Ok(Self) + } + + pub(crate) fn resume(&self, stack: &FiberStack, result: &Cell>) { + unsafe { + // Store where our result is going at the very tip-top of the + // stack, otherwise known as our reserved slot for this information. + // + // In the diagram above this is updating address 0xAff8 + let addr = stack.top().unwrap().cast::().offset(-1); + addr.write(result as *const _ as usize); + + wasmtime_fiber_switch(stack.top().unwrap()); + + // null this out to help catch use-after-free + addr.write(0); + } + } +} + +impl Suspend { + pub(crate) fn switch(&mut self, result: RunResult) -> A { + unsafe { + // Calculate 0xAff8 and then write to it + (*self.result_location::()).set(result); + + wasmtime_fiber_switch(self.top_of_stack); + + self.take_resume::() + } + } + + unsafe fn take_resume(&self) -> A { + match (*self.result_location::()).replace(RunResult::Executing) { + RunResult::Resuming(val) => val, + _ => panic!("not in resuming state"), + } + } + + unsafe fn result_location(&self) -> *const Cell> { + let ret = self.top_of_stack.cast::<*const u8>().offset(-1).read(); + assert!(!ret.is_null()); + ret.cast() + } +} diff --git a/crates/fiber/src/stackswitch.rs b/crates/fiber/src/stackswitch.rs new file mode 100644 index 000000000000..5e32bb989fb8 --- /dev/null +++ b/crates/fiber/src/stackswitch.rs @@ -0,0 +1,38 @@ +//! ISA-specific stack-switching routines. + +// The bodies are defined in inline assembly in the conditionally +// included modules below; their symbols are visible in the binary and +// accessed via the `extern "C"` declarations below that. + +cfg_if::cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + } else if #[cfg(target_arch = "x86_64")] { + mod x86_64; + } else if #[cfg(target_arch = "x86")] { + mod x86; + } else if #[cfg(target_arch = "arm")] { + mod arm; + } else if #[cfg(target_arch = "s390x")] { + // currently `global_asm!` isn't stable on s390x so this is an external + // assembler file built with the `build.rs`. + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + } else { + compile_error!("fibers are not supported on this CPU architecture"); + } +} + +extern "C" { + #[wasmtime_versioned_export_macros::versioned_link] + pub(crate) fn wasmtime_fiber_init( + top_of_stack: *mut u8, + entry: extern "C" fn(*mut u8, *mut u8), + entry_arg0: *mut u8, + ); + #[wasmtime_versioned_export_macros::versioned_link] + pub(crate) fn wasmtime_fiber_switch(top_of_stack: *mut u8); + #[allow(dead_code, reason = "only used on some platforms for inline asm")] + #[wasmtime_versioned_export_macros::versioned_link] + pub(crate) fn wasmtime_fiber_start(); +} diff --git a/crates/fiber/src/unix/aarch64.rs b/crates/fiber/src/stackswitch/aarch64.rs similarity index 100% rename from crates/fiber/src/unix/aarch64.rs rename to crates/fiber/src/stackswitch/aarch64.rs diff --git a/crates/fiber/src/unix/arm.rs b/crates/fiber/src/stackswitch/arm.rs similarity index 100% rename from crates/fiber/src/unix/arm.rs rename to crates/fiber/src/stackswitch/arm.rs diff --git a/crates/fiber/src/unix/riscv64.rs b/crates/fiber/src/stackswitch/riscv64.rs similarity index 100% rename from crates/fiber/src/unix/riscv64.rs rename to crates/fiber/src/stackswitch/riscv64.rs diff --git a/crates/fiber/src/unix/s390x.S b/crates/fiber/src/stackswitch/s390x.S similarity index 100% rename from crates/fiber/src/unix/s390x.S rename to crates/fiber/src/stackswitch/s390x.S diff --git a/crates/fiber/src/unix/x86.rs b/crates/fiber/src/stackswitch/x86.rs similarity index 100% rename from crates/fiber/src/unix/x86.rs rename to crates/fiber/src/stackswitch/x86.rs diff --git a/crates/fiber/src/unix/x86_64.rs b/crates/fiber/src/stackswitch/x86_64.rs similarity index 100% rename from crates/fiber/src/unix/x86_64.rs rename to crates/fiber/src/stackswitch/x86_64.rs diff --git a/crates/fiber/src/unix.rs b/crates/fiber/src/unix.rs index 65d6c78d5cff..2f320198865c 100644 --- a/crates/fiber/src/unix.rs +++ b/crates/fiber/src/unix.rs @@ -29,12 +29,16 @@ //! `suspend`, which has 0xB000 so it can find this, will read that and write //! its own resumption information into this slot as well. +use crate::stackswitch::*; use crate::{RunResult, RuntimeFiberStack}; +use std::boxed::Box; use std::cell::Cell; use std::io; use std::ops::Range; use std::ptr; +pub type Error = io::Error; + pub struct FiberStack { base: BasePtr, len: usize, @@ -192,20 +196,6 @@ pub struct Suspend { previous: asan::PreviousStack, } -extern "C" { - #[wasmtime_versioned_export_macros::versioned_link] - fn wasmtime_fiber_init( - top_of_stack: *mut u8, - entry: extern "C" fn(*mut u8, *mut u8), - entry_arg0: *mut u8, - ); - #[wasmtime_versioned_export_macros::versioned_link] - fn wasmtime_fiber_switch(top_of_stack: *mut u8); - #[allow(dead_code, reason = "only used on some platforms for inline asm")] - #[wasmtime_versioned_export_macros::versioned_link] - fn wasmtime_fiber_start(); -} - extern "C" fn fiber_start(arg0: *mut u8, top_of_stack: *mut u8) where F: FnOnce(A, &mut super::Suspend) -> C, @@ -288,25 +278,6 @@ impl Suspend { } } -cfg_if::cfg_if! { - if #[cfg(target_arch = "aarch64")] { - mod aarch64; - } else if #[cfg(target_arch = "x86_64")] { - mod x86_64; - } else if #[cfg(target_arch = "x86")] { - mod x86; - } else if #[cfg(target_arch = "arm")] { - mod arm; - } else if #[cfg(target_arch = "s390x")] { - // currently `global_asm!` isn't stable on s390x so this is an external - // assembler file built with the `build.rs`. - } else if #[cfg(target_arch = "riscv64")] { - mod riscv64; - } else { - compile_error!("fibers are not supported on this CPU architecture"); - } -} - /// Support for AddressSanitizer to support stack manipulations we do in this /// fiber implementation. /// @@ -481,6 +452,7 @@ mod asan { #[cfg(not(asan))] mod asan_disabled { use super::{FiberStack, RuntimeFiberStack}; + use std::boxed::Box; #[derive(Default)] pub struct PreviousStack; diff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rs index 16203bd5c926..2ce5f8464cc0 100644 --- a/crates/fiber/src/windows.rs +++ b/crates/fiber/src/windows.rs @@ -1,4 +1,5 @@ use crate::{RunResult, RuntimeFiberStack}; +use alloc::boxed::Box; use std::cell::Cell; use std::ffi::c_void; use std::io; @@ -7,6 +8,8 @@ use std::ptr; use windows_sys::Win32::Foundation::*; use windows_sys::Win32::System::Threading::*; +pub type Error = io::Error; + #[derive(Debug)] pub struct FiberStack(usize); diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 63c1eafbb20c..53497575eac2 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -185,7 +185,6 @@ async = [ "dep:async-trait", "wasmtime-component-macro?/async", "runtime", - "std", ] # Enables support for the pooling instance allocation strategy @@ -315,6 +314,7 @@ std = [ 'wasmtime-environ/std', 'object/std', 'once_cell', + 'wasmtime-fiber?/std', # technically this isn't necessary but once you have the standard library you # probably want things to go fast in which case you've probably got signal # handlers and such so implicitly enable this. This also helps reduce the diff --git a/crates/wasmtime/src/runtime/stack.rs b/crates/wasmtime/src/runtime/stack.rs index 94f5732ec40f..458558525c73 100644 --- a/crates/wasmtime/src/runtime/stack.rs +++ b/crates/wasmtime/src/runtime/stack.rs @@ -1,5 +1,6 @@ use crate::prelude::*; -use std::{ops::Range, sync::Arc}; +use alloc::sync::Arc; +use core::ops::Range; use wasmtime_fiber::{RuntimeFiberStack, RuntimeFiberStackCreator}; /// A stack creator. Can be used to provide a stack creator to wasmtime