code motion: cli related feature names (bytecodealliance#104)

* rename host runtime tests to command tests

* add a test for using wasi in a reactor

* commit cargo.lock

* reactor tests: fix wit deps

* test-programs: reactor-tests can build on stable

note that this fails because it exposes wasi-libc ctors calling import
functions from inside cabi_realloc

* test-programs: show that ctors fix in wit-bindgen fixes bug

* ci: install wasm32 targets for stable as well as nightly

* wit-bindgen: use 0.4.0

* ci: use wit-bindgen 0.4.0

* Co-habitate with wasi-common from wasmtime

* adapter: code motion in cargo feature & artifact names to cli-command, cli-reactor

there will shortly be a third type of reactor (non-cli, idk what to call it)

---------

Co-authored-by: Trevor Elliott <telliott@fastly.com>

host/tests/runtime.rs → host/tests/command.rs

@@ -18,7 +18,7 @@ use wasmtime::{
     Config, Engine, Store,
 };
 
-test_programs_macros::tests!();
+test_programs_macros::command_tests!();
 
 // A bunch of these test cases are expected to fail. We wrap up their execution in this
 // function so that we see if changes make them start passing.

host/tests/reactor.rs

@@ -0,0 +1,62 @@
+use anyhow::Result;
+use host::{add_to_linker, WasiCtx};
+use wasi_cap_std_sync::WasiCtxBuilder;
+use wasmtime::{
+    component::{Component, Linker},
+    Config, Engine, Store,
+};
+test_programs_macros::reactor_tests!();
+
+wasmtime::component::bindgen!({
+    path: "../test-programs/reactor-tests/wit",
+    world: "test-reactor",
+    async: true,
+});
+
+async fn instantiate(path: &str) -> Result<(Store<WasiCtx>, TestReactor)> {
+    println!("{}", path);
+
+    let mut config = Config::new();
+    config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
+    config.wasm_component_model(true);
+    config.async_support(true);
+
+    let engine = Engine::new(&config)?;
+    let component = Component::from_file(&engine, &path)?;
+    let mut linker = Linker::new(&engine);
+    add_to_linker(&mut linker, |x| x)?;
+
+    let mut store = Store::new(&engine, WasiCtxBuilder::new().build());
+
+    let (wasi, _instance) = TestReactor::instantiate_async(&mut store, &component, &linker).await?;
+    Ok((store, wasi))
+}
+
+async fn run_reactor_tests(mut store: Store<WasiCtx>, reactor: TestReactor) -> Result<()> {
+    store
+        .data_mut()
+        .env
+        .push(("GOOD_DOG".to_owned(), "gussie".to_owned()));
+
+    let r = reactor
+        .call_add_strings(&mut store, &["hello", "$GOOD_DOG"])
+        .await?;
+    assert_eq!(r, 2);
+
+    // Redefine the env, show that the adapter only fetches it once
+    // even if the libc ctors copy it in multiple times:
+    store.data_mut().env.clear();
+    store
+        .data_mut()
+        .env
+        .push(("GOOD_DOG".to_owned(), "cody".to_owned()));
+    // Cody is indeed good but this should be "hello again" "gussie"
+    let r = reactor
+        .call_add_strings(&mut store, &["hello again", "$GOOD_DOG"])
+        .await?;
+    assert_eq!(r, 4);
+
+    let contents = reactor.call_get_strings(&mut store).await?;
+    assert_eq!(contents, &["hello", "gussie", "hello again", "gussie"]);
+    Ok(())
+}

test-programs/Cargo.toml

@@ -8,4 +8,4 @@ publish = false
 getrandom = "0.2.8"
 rustix = "0.36.6"
 cap-std = "1.0.3"
-wit-bindgen = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "39030f9adabe581ceae7d574e4a902e37d1be6eb" }
+wit-bindgen = "0.4.0"

test-programs/macros/Cargo.toml

@@ -14,3 +14,5 @@ quote = "1.0"
 
 [build-dependencies]
 wit-component = "0.7.0"
+cargo_metadata = "0.15.3"
+heck = "0.4.0"

test-programs/macros/build.rs

@@ -1,72 +1,98 @@
+use heck::ToSnakeCase;
 use std::env;
 use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
 use wit_component::ComponentEncoder;
 
-fn main() {
+fn build_adapter(name: &str, features: &[&str]) -> Vec<u8> {
     let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
 
-    let mut components = Vec::new();
-
     println!("cargo:rerun-if-changed=../../src");
     let mut cmd = Command::new("cargo");
     cmd.arg("build")
         .arg("--release")
         .current_dir("../../")
         .arg("--target=wasm32-unknown-unknown")
-        .arg("--features=command")
         .env("CARGO_TARGET_DIR", &out_dir)
         .env_remove("CARGO_ENCODED_RUSTFLAGS");
+    for f in features {
+        cmd.arg(f);
+    }
     let status = cmd.status().unwrap();
     assert!(status.success());
 
-    let wasi_adapter = out_dir.join("wasm32-unknown-unknown/release/wasi_snapshot_preview1.wasm");
-    println!("wasi adapter: {:?}", &wasi_adapter);
-    let wasi_adapter = fs::read(&wasi_adapter).unwrap();
+    let adapter = out_dir.join(format!("wasi_snapshot_preview1.{name}.wasm"));
+    std::fs::copy(
+        out_dir.join("wasm32-unknown-unknown/release/wasi_snapshot_preview1.wasm"),
+        &adapter,
+    )
+    .unwrap();
+    println!("wasi {name} adapter: {:?}", &adapter);
+    fs::read(&adapter).unwrap()
+}
 
+fn main() {
+    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
+
+    let cli_reactor_adapter = build_adapter("cli_reactor", &[]);
+    let cli_command_adapter = build_adapter(
+        "cli_command",
+        &["--no-default-features", "--features=cli-command"],
+    );
+
+    // Build all test program crates
+    // wasi-tests and test-programs require nightly for a feature in the `errno` crate
     println!("cargo:rerun-if-changed=..");
     let mut cmd = Command::new("rustup");
     cmd.arg("run")
         .arg("nightly")
         .arg("cargo")
         .arg("build")
         .arg("--target=wasm32-wasi")
+        .arg("--package=wasi-tests")
+        .arg("--package=test-programs")
         .current_dir("..")
         .env("CARGO_TARGET_DIR", &out_dir)
         .env("CARGO_PROFILE_DEV_DEBUG", "1")
         .env_remove("CARGO_ENCODED_RUSTFLAGS");
     let status = cmd.status().unwrap();
     assert!(status.success());
 
+    // reactor-tests can build on stable:
     let mut cmd = Command::new("rustup");
     cmd.arg("run")
-        .arg("nightly")
+        .arg("stable")
         .arg("cargo")
         .arg("build")
         .arg("--target=wasm32-wasi")
-        .arg("--package=wasi-tests")
-        .arg("--package=test-programs")
+        .arg("--package=reactor-tests")
         .current_dir("..")
         .env("CARGO_TARGET_DIR", &out_dir)
         .env("CARGO_PROFILE_DEV_DEBUG", "1")
         .env_remove("CARGO_ENCODED_RUSTFLAGS");
     let status = cmd.status().unwrap();
     assert!(status.success());
 
-    for file in out_dir.join("wasm32-wasi/debug").read_dir().unwrap() {
-        let file = file.unwrap().path();
-        if file.extension().and_then(|s| s.to_str()) != Some("wasm") {
-            continue;
-        }
-        let stem = file.file_stem().unwrap().to_str().unwrap().to_string();
+    let meta = cargo_metadata::MetadataCommand::new().exec().unwrap();
+
+    let mut command_components = Vec::new();
+
+    for stem in targets_in_package(&meta, "test-programs", "bin").chain(targets_in_package(
+        &meta,
+        "wasi-tests",
+        "bin",
+    )) {
+        let file = out_dir
+            .join("wasm32-wasi/debug")
+            .join(format!("{stem}.wasm"));
 
         let module = fs::read(&file).expect("read wasm module");
         let component = ComponentEncoder::default()
             .module(module.as_slice())
             .unwrap()
             .validate(true)
-            .adapter("wasi_snapshot_preview1", &wasi_adapter)
+            .adapter("wasi_snapshot_preview1", &cli_command_adapter)
             .unwrap()
             .encode()
             .expect(&format!(
@@ -75,9 +101,53 @@ fn main() {
             ));
         let component_path = out_dir.join(format!("{}.component.wasm", &stem));
         fs::write(&component_path, component).expect("write component to disk");
-        components.push((stem, component_path));
+        command_components.push((stem, component_path));
     }
 
-    let src = format!("const COMPONENTS: &[(&str, &str)] = &{:?};", components);
+    let mut reactor_components = Vec::new();
+
+    for stem in targets_in_package(&meta, "reactor-tests", "cdylib") {
+        let stem = stem.to_snake_case();
+        let file = out_dir
+            .join("wasm32-wasi/debug")
+            .join(format!("{stem}.wasm"));
+
+        let module = fs::read(&file).expect(&format!("read wasm module: {file:?}"));
+        let component = ComponentEncoder::default()
+            .module(module.as_slice())
+            .unwrap()
+            .validate(true)
+            .adapter("wasi_snapshot_preview1", &cli_reactor_adapter)
+            .unwrap()
+            .encode()
+            .expect(&format!(
+                "module {:?} can be translated to a component",
+                file
+            ));
+        let component_path = out_dir.join(format!("{}.component.wasm", &stem));
+        fs::write(&component_path, component).expect("write component to disk");
+        reactor_components.push((stem, component_path));
+    }
+
+    let src = format!(
+        "const COMMAND_COMPONENTS: &[(&str, &str)] = &{command_components:?};
+         const REACTOR_COMPONENTS: &[(&str, &str)] = &{reactor_components:?};
+        ",
+    );
     std::fs::write(out_dir.join("components.rs"), src).unwrap();
 }
+
+fn targets_in_package<'a>(
+    meta: &'a cargo_metadata::Metadata,
+    package: &'a str,
+    kind: &'a str,
+) -> impl Iterator<Item = &'a String> + 'a {
+    meta.packages
+        .iter()
+        .find(|p| p.name == package)
+        .unwrap()
+        .targets
+        .iter()
+        .filter(move |t| t.kind == &[kind])
+        .map(|t| &t.name)
+}

test-programs/macros/src/lib.rs

@@ -4,8 +4,24 @@ use quote::quote;
 include!(concat!(env!("OUT_DIR"), "/components.rs"));
 
 #[proc_macro]
-pub fn tests(_input: TokenStream) -> TokenStream {
-    let tests = COMPONENTS.iter().map(|(stem, file)| {
+pub fn command_tests(_input: TokenStream) -> TokenStream {
+    let tests = COMMAND_COMPONENTS.iter().map(|(stem, file)| {
+        let name = quote::format_ident!("{}", stem);
+        let runner = quote::format_ident!("run_{}", stem);
+        quote! {
+            #[test_log::test(tokio::test)]
+            async fn #name() -> anyhow::Result<()> {
+                let (store, inst) = instantiate(#file).await?;
+                #runner(store, inst).await
+            }
+        }
+    });
+    quote!(#(#tests)*).into()
+}
+
+#[proc_macro]
+pub fn reactor_tests(_input: TokenStream) -> TokenStream {
+    let tests = REACTOR_COMPONENTS.iter().map(|(stem, file)| {
         let name = quote::format_ident!("{}", stem);
         let runner = quote::format_ident!("run_{}", stem);
         quote! {

test-programs/reactor-tests/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "reactor-tests"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+crate-type=["cdylib"]
+
+[dependencies]
+wit-bindgen = "0.4.0"

test-programs/reactor-tests/src/lib.rs

@@ -0,0 +1,25 @@
+wit_bindgen::generate!("test-reactor");
+
+export_test_reactor!(T);
+
+struct T;
+
+static mut STATE: Vec<String> = Vec::new();
+
+impl TestReactor for T {
+    fn add_strings(ss: Vec<String>) -> u32 {
+        for s in ss {
+            match s.split_once("$") {
+                Some((prefix, var)) if prefix.is_empty() => match std::env::var(var) {
+                    Ok(val) => unsafe { STATE.push(val) },
+                    Err(_) => unsafe { STATE.push("undefined".to_owned()) },
+                },
+                _ => unsafe { STATE.push(s) },
+            }
+        }
+        unsafe { STATE.len() as u32 }
+    }
+    fn get_strings() -> Vec<String> {
+        unsafe { STATE.clone() }
+    }
+}

test-programs/reactor-tests/wit/deps/clocks

@@ -0,0 +1 @@
+../../../../wit/deps/clocks

test-programs/reactor-tests/wit/deps/filesystem

@@ -0,0 +1 @@
+../../../../wit/deps/filesystem

test-programs/reactor-tests/wit/deps/io

@@ -0,0 +1 @@
+../../../../wit/deps/io

test-programs/reactor-tests/wit/deps/poll

@@ -0,0 +1 @@
+../../../../wit/deps/poll

test-programs/reactor-tests/wit/deps/wasi/environment-preopens.wit

@@ -0,0 +1 @@
+../../../../../wit/environment-preopens.wit

test-programs/reactor-tests/wit/deps/wasi/environment.wit

@@ -0,0 +1 @@
+../../../../../wit/environment.wit

test-programs/reactor-tests/wit/test-reactor.wit

@@ -0,0 +1,7 @@
+default world test-reactor {
+
+  import wasi-environment: wasi.environment
+
+  export add-strings: func(s: list<string>) -> u32
+  export get-strings: func() -> list<string>
+}