From 2d34dbef4b24ced62b5f6bb461c0caa9eab9783e Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Wed, 5 Apr 2023 13:33:03 -0700 Subject: [PATCH] Begin implementation of wasi-http (#5929) * Integrate experimental HTTP into wasmtime. * Reset Cargo.lock * Switch to bail!, plumb options partially. * Implement timeouts. * Remove generated files & wasm, add Makefile * Remove generated code textfile * Update crates/wasi-http/Cargo.toml Co-authored-by: Eduardo de Moura Rodrigues <16357187+eduardomourar@users.noreply.github.com> * Update crates/wasi-http/Cargo.toml Co-authored-by: Eduardo de Moura Rodrigues <16357187+eduardomourar@users.noreply.github.com> * Extract streams from request/response. * Fix read for len < buffer length. * Formatting. * types impl: swap todos for traps * streams_impl: idioms, and swap todos for traps * component impl: idioms, swap all unwraps for traps, swap all todos for traps * http impl: idiom * Remove an unnecessary mut. * Remove an unsupported function. * Switch to the tokio runtime for the HTTP request. * Add a rust example. * Update to latest wit definition * Remove example code. * wip: start writing a http test... * finish writing the outbound request example havent executed it yet * better debug output * wasi-http: some stubs required for rust rewrite of the example * add wasi_http tests to test-programs * CI: run the http tests * Fix some warnings. * bump new deps to latest releases (#3) * Add tests for wasi-http to test-programs (#2) * wip: start writing a http test... * finish writing the outbound request example havent executed it yet * better debug output * wasi-http: some stubs required for rust rewrite of the example * add wasi_http tests to test-programs * CI: run the http tests * bump new deps to latest releases h2 0.3.16 http 0.2.9 mio 0.8.6 openssl 0.10.48 openssl-sys 0.9.83 tokio 1.26.0 --------- Co-authored-by: Brendan Burns * Update crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs * Update crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs * Update crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs * wasi-http: fix cargo.toml file and publish script to work together (#4) unfortunately, the publish script doesn't use a proper toml parser (in order to not have any dependencies), so the whitespace has to be the trivial expected case. then, add wasi-http to the list of crates to publish. * Update crates/test-programs/build.rs * Switch to rustls * Cleanups. * Merge switch to rustls. * Formatting * Remove libssl install * Fix tests. * Rename wasi-http -> wasmtime-wasi-http * prtest:full Conditionalize TLS on riscv64gc. * prtest:full Fix formatting, also disable tls on s390x * prtest:full Add a path parameter to wit-bindgen, remove symlink. * prtest:full Fix tests for places where SSL isn't supported. * Update crates/wasi-http/Cargo.toml --------- Co-authored-by: Eduardo de Moura Rodrigues <16357187+eduardomourar@users.noreply.github.com> Co-authored-by: Pat Hickey Co-authored-by: Pat Hickey --- .github/workflows/main.yml | 1 + .gitmodules | 3 + Cargo.lock | 342 ++++++++++++-- Cargo.toml | 6 + ci/run-tests.sh | 1 + crates/cli-flags/src/lib.rs | 22 +- crates/test-programs/Cargo.toml | 3 + crates/test-programs/build.rs | 266 +++++++---- crates/test-programs/tests/http_tests/main.rs | 12 + .../tests/http_tests/runtime/mod.rs | 1 + .../http_tests/runtime/wasi_http_tests.rs | 39 ++ .../test-programs/tests/http_tests/utils.rs | 13 + .../test-programs/wasi-http-tests/Cargo.lock | 380 +++++++++++++++ .../test-programs/wasi-http-tests/Cargo.toml | 15 + .../src/bin/outbound_request.rs | 132 ++++++ .../test-programs/wasi-http-tests/src/lib.rs | 3 + crates/wasi-http/Cargo.toml | 26 ++ crates/wasi-http/src/component_impl.rs | 431 ++++++++++++++++++ crates/wasi-http/src/http_impl.rs | 211 +++++++++ crates/wasi-http/src/lib.rs | 27 ++ crates/wasi-http/src/streams_impl.rs | 93 ++++ crates/wasi-http/src/struct.rs | 80 ++++ crates/wasi-http/src/types_impl.rs | 271 +++++++++++ crates/wasi-http/wasi-http | 1 + scripts/publish.rs | 1 + src/commands/run.rs | 27 +- 26 files changed, 2284 insertions(+), 123 deletions(-) create mode 100644 crates/test-programs/tests/http_tests/main.rs create mode 100644 crates/test-programs/tests/http_tests/runtime/mod.rs create mode 100644 crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs create mode 100644 crates/test-programs/tests/http_tests/utils.rs create mode 100644 crates/test-programs/wasi-http-tests/Cargo.lock create mode 100644 crates/test-programs/wasi-http-tests/Cargo.toml create mode 100644 crates/test-programs/wasi-http-tests/src/bin/outbound_request.rs create mode 100644 crates/test-programs/wasi-http-tests/src/lib.rs create mode 100644 crates/wasi-http/Cargo.toml create mode 100644 crates/wasi-http/src/component_impl.rs create mode 100644 crates/wasi-http/src/http_impl.rs create mode 100644 crates/wasi-http/src/lib.rs create mode 100644 crates/wasi-http/src/streams_impl.rs create mode 100644 crates/wasi-http/src/struct.rs create mode 100644 crates/wasi-http/src/types_impl.rs create mode 160000 crates/wasi-http/wasi-http diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15d2fa4c1833..a4704d0108d6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -387,6 +387,7 @@ jobs: - run: cargo fetch --locked - run: cargo fetch --locked --manifest-path crates/test-programs/wasi-tests/Cargo.toml + - run: cargo fetch --locked --manifest-path crates/test-programs/wasi-http-tests/Cargo.toml - uses: actions/cache@v3 with: diff --git a/.gitmodules b/.gitmodules index ba7f6793433a..f361970532fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "tests/wasi_testsuite/wasi-threads"] path = tests/wasi_testsuite/wasi-threads url = https://github.com/WebAssembly/wasi-threads +[submodule "crates/wasi-http/wasi-http"] + path = crates/wasi-http/wasi-http + url = https://github.com/WebAssembly/wasi-http diff --git a/Cargo.lock b/Cargo.lock index da38baa7eaf6..acd7c6d70f9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,6 +240,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "camino" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c530edf18f37068ac2d977409ed5cd50d53d73bc653c7647b48eb78976ac9ae2" +dependencies = [ + "serde", +] + [[package]] name = "cap-fs-ext" version = "1.0.5" @@ -336,6 +345,29 @@ dependencies = [ "libc", ] +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cast" version = "0.3.0" @@ -1315,6 +1347,45 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" + +[[package]] +name = "futures-sink" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" + +[[package]] +name = "futures-task" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" + +[[package]] +name = "futures-util" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -1394,6 +1465,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1465,6 +1555,52 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951dfc2e32ac02d67c90c0d65bd27009a635dc9b381a2cc7d284ab01e3a0150d" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92445bc9cc14bfa0a3ce56817dc3b5bcc227a168781a356b702410789cec0d10" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "1.3.0" @@ -1480,6 +1616,28 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "1.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75264b2003a3913f118d35c586e535293b3e22e41f074930762929d071e092" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "tracing", + "want", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -1808,34 +1966,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -2074,6 +2212,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs1" version = "0.2.4" @@ -2487,6 +2631,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rsa" version = "0.5.0" @@ -2530,6 +2689,28 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07180898a28ed6a7f7ba2311594308f595e3dd2e3c3812fa0a80a47b45f17e5d" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rusty-fork" version = "0.3.0" @@ -2563,6 +2744,25 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +dependencies = [ + "serde", +] + [[package]] name = "serde" version = "1.0.137" @@ -2664,6 +2864,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e24979f63a11545f5f2c60141afe249d4f19f84581ea2138065e400941d83d3" +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "slice-group-by" version = "0.3.0" @@ -2681,9 +2890,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi", @@ -2833,6 +3042,7 @@ version = "0.0.0" dependencies = [ "anyhow", "cap-std", + "cargo_metadata", "cfg-if", "os_pipe", "target-lexicon", @@ -2843,6 +3053,7 @@ dependencies = [ "wasi-common", "wasmtime", "wasmtime-wasi", + "wasmtime-wasi-http", "wat", ] @@ -2908,10 +3119,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.4" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bfb875c82dc0a4f1f37a30e720dee181a2b3a06a428b0fc6873ea38d6407850" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ + "autocfg 1.1.0", "bytes", "libc", "memchr", @@ -2920,7 +3132,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "winapi", + "windows-sys", ] [[package]] @@ -2934,6 +3146,30 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.9" @@ -2988,6 +3224,12 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.15.0" @@ -3040,6 +3282,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -3105,6 +3353,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3551,6 +3809,7 @@ dependencies = [ "wasmtime-runtime", "wasmtime-wasi", "wasmtime-wasi-crypto", + "wasmtime-wasi-http", "wasmtime-wasi-nn", "wasmtime-wasi-threads", "wasmtime-wast", @@ -3842,6 +4101,24 @@ dependencies = [ "wiggle", ] +[[package]] +name = "wasmtime-wasi-http" +version = "0.0.1" +dependencies = [ + "anyhow", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "rustls", + "thiserror", + "tokio", + "tokio-rustls", + "wasmtime", + "webpki-roots", +] + [[package]] name = "wasmtime-wasi-nn" version = "9.0.0" @@ -3940,6 +4217,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" +dependencies = [ + "rustls-webpki", +] + [[package]] name = "which" version = "4.2.5" diff --git a/Cargo.toml b/Cargo.toml index 03423e8b8e80..e2362f3bd530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ wasmtime-wasi = { workspace = true, features = ["exit"] } wasmtime-wasi-crypto = { workspace = true, optional = true } wasmtime-wasi-nn = { workspace = true, optional = true } wasmtime-wasi-threads = { workspace = true, optional = true } +wasmtime-wasi-http = { workspace = true, optional = true } clap = { workspace = true, features = ["color", "suggestions", "derive"] } anyhow = { workspace = true } target-lexicon = { workspace = true } @@ -165,6 +166,8 @@ cranelift-bforest = { path = "cranelift/bforest", version = "0.96.0" } cranelift-control = { path = "cranelift/control", version = "0.96.0" } cranelift = { path = "cranelift/umbrella", version = "0.96.0" } +wasmtime-wasi-http = { path = "crates/wasi-http", version = "=0.0.1" } + winch-codegen = { path = "winch/codegen", version = "=0.7.0" } winch-environ = { path = "winch/environ", version = "=0.7.0" } winch-filetests = { path = "winch/filetests" } @@ -213,6 +216,7 @@ default = [ "vtune", "wasi-nn", "wasi-threads", + "wasi-http", "pooling-allocator", ] jitdump = ["wasmtime/jitdump"] @@ -220,6 +224,7 @@ vtune = ["wasmtime/vtune"] wasi-crypto = ["dep:wasmtime-wasi-crypto"] wasi-nn = ["dep:wasmtime-wasi-nn"] wasi-threads = ["dep:wasmtime-wasi-threads"] +wasi-http = ["dep:wasmtime-wasi-http"] pooling-allocator = ["wasmtime/pooling-allocator", "wasmtime-cli-flags/pooling-allocator"] all-arch = ["wasmtime/all-arch"] posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"] @@ -264,3 +269,4 @@ harness = false [[bench]] name = "wasi" harness = false + diff --git a/ci/run-tests.sh b/ci/run-tests.sh index db187b779e40..67e53eb87b51 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -2,6 +2,7 @@ cargo test \ --features "test-programs/test_programs" \ + --features "test-programs/test_programs_http" \ --features wasi-threads \ --workspace \ --exclude 'wasmtime-wasi-*' \ diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 538f39d67834..495021fbc786 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -66,6 +66,10 @@ pub const SUPPORTED_WASI_MODULES: &[(&str, &str)] = &[ "experimental-wasi-threads", "enables support for the WASI threading API (experimental), see https://github.com/WebAssembly/wasi-threads", ), + ( + "experimental-wasi-http", + "enables support for the WASI HTTP APIs (experimental), see https://github.com/WebAssembly/wasi-http", + ), ]; fn init_file_per_thread_logger(prefix: &'static str) { @@ -502,6 +506,7 @@ fn parse_wasi_modules(modules: &str) -> Result { "experimental-wasi-crypto" => Ok(wasi_modules.wasi_crypto = enable), "experimental-wasi-nn" => Ok(wasi_modules.wasi_nn = enable), "experimental-wasi-threads" => Ok(wasi_modules.wasi_threads = enable), + "experimental-wasi-http" => Ok(wasi_modules.wasi_http = enable), "default" => bail!("'default' cannot be specified with other WASI modules"), _ => bail!("unsupported WASI module '{}'", module), }; @@ -536,6 +541,9 @@ pub struct WasiModules { /// Enable the experimental wasi-threads implementation. pub wasi_threads: bool, + + /// Enable the experimental wasi-http implementation + pub wasi_http: bool, } impl Default for WasiModules { @@ -545,6 +553,7 @@ impl Default for WasiModules { wasi_crypto: false, wasi_nn: false, wasi_threads: false, + wasi_http: false, } } } @@ -557,6 +566,7 @@ impl WasiModules { wasi_nn: false, wasi_crypto: false, wasi_threads: false, + wasi_http: false, } } } @@ -711,7 +721,8 @@ mod test { wasi_common: true, wasi_crypto: false, wasi_nn: false, - wasi_threads: false + wasi_threads: false, + wasi_http: false, } ); } @@ -725,7 +736,8 @@ mod test { wasi_common: true, wasi_crypto: false, wasi_nn: false, - wasi_threads: false + wasi_threads: false, + wasi_http: false } ); } @@ -743,7 +755,8 @@ mod test { wasi_common: false, wasi_crypto: false, wasi_nn: true, - wasi_threads: false + wasi_threads: false, + wasi_http: false, } ); } @@ -758,7 +771,8 @@ mod test { wasi_common: false, wasi_crypto: false, wasi_nn: false, - wasi_threads: false + wasi_threads: false, + wasi_http: false, } ); } diff --git a/crates/test-programs/Cargo.toml b/crates/test-programs/Cargo.toml index a9a30df23203..99f893f17467 100644 --- a/crates/test-programs/Cargo.toml +++ b/crates/test-programs/Cargo.toml @@ -9,6 +9,7 @@ license = "Apache-2.0 WITH LLVM-exception" [build-dependencies] cfg-if = "1.0" +cargo_metadata = "0.15.3" [dev-dependencies] wasi-common = { workspace = true } @@ -23,6 +24,8 @@ anyhow = { workspace = true } wat = { workspace = true } cap-std = { workspace = true } tokio = { version = "1.8.0", features = ["rt-multi-thread"] } +wasmtime-wasi-http = { workspace = true } [features] test_programs = [] +test_programs_http = [ "wasmtime/component-model" ] diff --git a/crates/test-programs/build.rs b/crates/test-programs/build.rs index 55e12aa0b51b..60c010aa01bd 100644 --- a/crates/test-programs/build.rs +++ b/crates/test-programs/build.rs @@ -1,25 +1,105 @@ +#![allow(dead_code, unused_imports)] //! Build program to generate a program which runs all the testsuites. //! //! By generating a separate `#[test]` test for each file, we allow cargo test //! to automatically run the files in parallel. +use std::fs::{read_dir, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; fn main() { #[cfg(feature = "test_programs")] - wasi_tests::build_and_generate_tests() + wasi_tests::build_and_generate_tests(); + #[cfg(feature = "test_programs_http")] + wasi_http_tests::build_and_generate_tests(); +} + +fn build_tests(testsuite: &str, out_dir: &Path) -> io::Result> { + let mut cmd = Command::new("cargo"); + cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "1"); + cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); + cmd.args(&[ + "build", + "--release", + "--target=wasm32-wasi", + "--target-dir", + out_dir.to_str().unwrap(), + ]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .current_dir(testsuite); + let output = cmd.output()?; + + let status = output.status; + if !status.success() { + panic!( + "Building tests failed: exit code: {}", + status.code().unwrap() + ); + } + + let meta = cargo_metadata::MetadataCommand::new() + .manifest_path(PathBuf::from(testsuite).join("Cargo.toml")) + .exec() + .expect("cargo metadata"); + + Ok(meta + .packages + .iter() + .find(|p| p.name == testsuite) + .unwrap() + .targets + .iter() + .filter(|t| t.kind == ["bin"]) + .map(|t| t.name.clone()) + .collect::>()) +} + +#[allow(dead_code)] +fn test_directory( + out: &mut File, + test_binaries: &[String], + testsuite: &str, + runtime: &str, + out_dir: &Path, + mut write_testsuite_tests: impl FnMut(&mut File, &Path, &str) -> io::Result<()>, +) -> io::Result<()> { + writeln!( + out, + "mod {} {{", + Path::new(testsuite) + .file_stem() + .expect("testsuite filename should have a stem") + .to_str() + .expect("testsuite filename should be representable as a string") + .replace("-", "_") + )?; + writeln!( + out, + " use super::{{runtime::{} as runtime, utils, setup_log}};", + runtime + )?; + for test_binary in test_binaries { + let binary_path = out_dir + .join("wasm32-wasi") + .join("release") + .join(format!("{}.wasm", test_binary.replace("-", "_"))); + write_testsuite_tests(out, &binary_path, testsuite)?; + } + writeln!(out, "}}")?; + Ok(()) } #[cfg(feature = "test_programs")] mod wasi_tests { + use super::*; use std::env; - use std::fs::{read_dir, File}; - use std::io::{self, Write}; - use std::path::{Path, PathBuf}; - use std::process::{Command, Stdio}; pub(super) fn build_and_generate_tests() { // Validate if any of test sources are present and if they changed // This should always work since there is no submodule to init anymore - let bin_tests = std::fs::read_dir("wasi-tests/src/bin").unwrap(); + let bin_tests = read_dir("wasi-tests/src/bin").unwrap(); for test in bin_tests { if let Ok(test_file) = test { let test_file_path = test_file @@ -38,90 +118,25 @@ mod wasi_tests { ); let mut out = File::create(out_dir.join("wasi_tests.rs")).expect("error generating test source file"); - build_tests("wasi-tests", &out_dir).expect("building tests"); - test_directory(&mut out, "wasi-cap-std-sync", "cap_std_sync", &out_dir) - .expect("generating wasi-cap-std-sync tests"); - test_directory(&mut out, "wasi-tokio", "tokio", &out_dir) - .expect("generating wasi-tokio tests"); - } - - fn build_tests(testsuite: &str, out_dir: &Path) -> io::Result<()> { - let mut cmd = Command::new("cargo"); - cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "1"); - cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); - cmd.args(&[ - "build", - "--release", - "--target=wasm32-wasi", - "--target-dir", - out_dir.to_str().unwrap(), - ]) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .current_dir(testsuite); - let output = cmd.output()?; - - let status = output.status; - if !status.success() { - panic!( - "Building tests failed: exit code: {}", - status.code().unwrap() - ); - } - - Ok(()) - } - - fn test_directory( - out: &mut File, - testsuite: &str, - runtime: &str, - out_dir: &Path, - ) -> io::Result<()> { - let mut dir_entries: Vec<_> = read_dir(out_dir.join("wasm32-wasi/release")) - .expect("reading testsuite directory") - .map(|r| r.expect("reading testsuite directory entry")) - .filter(|dir_entry| { - let p = dir_entry.path(); - if let Some(ext) = p.extension() { - // Only look at wast files. - if ext == "wasm" { - // Ignore files starting with `.`, which could be editor temporary files - if let Some(stem) = p.file_stem() { - if let Some(stemstr) = stem.to_str() { - if !stemstr.starts_with('.') { - return true; - } - } - } - } - } - false - }) - .collect(); - - dir_entries.sort_by_key(|dir| dir.path()); - - writeln!( - out, - "mod {} {{", - Path::new(testsuite) - .file_stem() - .expect("testsuite filename should have a stem") - .to_str() - .expect("testsuite filename should be representable as a string") - .replace("-", "_") - )?; - writeln!( - out, - " use super::{{runtime::{} as runtime, utils, setup_log}};", - runtime - )?; - for dir_entry in dir_entries { - write_testsuite_tests(out, &dir_entry.path(), testsuite)?; - } - writeln!(out, "}}")?; - Ok(()) + let test_binaries = build_tests("wasi-tests", &out_dir).expect("building tests"); + test_directory( + &mut out, + &test_binaries, + "wasi-cap-std-sync", + "cap_std_sync", + &out_dir, + write_testsuite_tests, + ) + .expect("generating wasi-cap-std-sync tests"); + test_directory( + &mut out, + &test_binaries, + "wasi-tokio", + "tokio", + &out_dir, + write_testsuite_tests, + ) + .expect("generating wasi-tokio tests"); } fn write_testsuite_tests(out: &mut File, path: &Path, testsuite: &str) -> io::Result<()> { @@ -277,3 +292,74 @@ mod wasi_tests { } } } + +#[cfg(feature = "test_programs_http")] +mod wasi_http_tests { + use super::*; + use std::env; + + pub(super) fn build_and_generate_tests() { + // Validate if any of test sources are present and if they changed + // This should always work since there is no submodule to init anymore + let bin_tests = read_dir("wasi-http-tests/src/bin").unwrap(); + for test in bin_tests { + if let Ok(test_file) = test { + let test_file_path = test_file + .path() + .into_os_string() + .into_string() + .expect("test file path"); + println!("cargo:rerun-if-changed={}", test_file_path); + } + } + println!("cargo:rerun-if-changed=wasi-http-tests/Cargo.toml"); + println!("cargo:rerun-if-changed=wasi-http-tests/src/lib.rs"); + // Build tests to OUT_DIR (target/*/build/wasi-common-*/out/wasm32-wasi/release/*.wasm) + let out_dir = PathBuf::from( + env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set"), + ); + let mut out = File::create(out_dir.join("wasi_http_tests.rs")) + .expect("error generating test source file"); + + let test_binaries = build_tests("wasi-http-tests", &out_dir).expect("building tests"); + test_directory( + &mut out, + &test_binaries, + "wasi-http-tests", + "wasi_http_tests", + &out_dir, + write_testsuite_tests, + ) + .expect("generating wasi-cap-std-sync tests"); + } + + fn write_testsuite_tests(out: &mut File, path: &Path, _testsuite: &str) -> io::Result<()> { + let stemstr = path + .file_stem() + .expect("file_stem") + .to_str() + .expect("to_str"); + + writeln!(out, " #[test]")?; + let test_fn_name = stemstr.replace("-", "_"); + writeln!(out, " fn r#{}() -> anyhow::Result<()> {{", test_fn_name,)?; + writeln!(out, " setup_log();")?; + writeln!( + out, + " let path = std::path::Path::new(r#\"{}\"#);", + path.display() + )?; + writeln!(out, " let data = wat::parse_file(path)?;")?; + writeln!( + out, + " let bin_name = utils::extract_exec_name_from_path(path)?;" + )?; + writeln!( + out, + " runtime::instantiate_inherit_stdio(&data, &bin_name, None)", + )?; + writeln!(out, " }}")?; + writeln!(out)?; + Ok(()) + } +} diff --git a/crates/test-programs/tests/http_tests/main.rs b/crates/test-programs/tests/http_tests/main.rs new file mode 100644 index 000000000000..28a8c956d9f4 --- /dev/null +++ b/crates/test-programs/tests/http_tests/main.rs @@ -0,0 +1,12 @@ +#![cfg(feature = "test_programs_http")] +use std::sync::Once; +mod runtime; +mod utils; + +static LOG_INIT: Once = Once::new(); + +fn setup_log() { + LOG_INIT.call_once(tracing_subscriber::fmt::init) +} + +include!(concat!(env!("OUT_DIR"), "/wasi_http_tests.rs")); diff --git a/crates/test-programs/tests/http_tests/runtime/mod.rs b/crates/test-programs/tests/http_tests/runtime/mod.rs new file mode 100644 index 000000000000..5af38c5cb518 --- /dev/null +++ b/crates/test-programs/tests/http_tests/runtime/mod.rs @@ -0,0 +1 @@ +pub mod wasi_http_tests; diff --git a/crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs b/crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs new file mode 100644 index 000000000000..e322b6d799e4 --- /dev/null +++ b/crates/test-programs/tests/http_tests/runtime/wasi_http_tests.rs @@ -0,0 +1,39 @@ +use anyhow::Context; +use std::path::Path; +use wasmtime::{Config, Engine, Linker, Module, Store}; +use wasmtime_wasi::{sync::WasiCtxBuilder, WasiCtx}; +use wasmtime_wasi_http::WasiHttp; + +pub fn instantiate_inherit_stdio( + data: &[u8], + bin_name: &str, + _workspace: Option<&Path>, +) -> anyhow::Result<()> { + let config = Config::new(); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, &data).context("failed to create wasm module")?; + let mut linker = Linker::new(&engine); + + struct Ctx { + wasi: WasiCtx, + http: WasiHttp, + } + + wasmtime_wasi::sync::add_to_linker(&mut linker, |cx: &mut Ctx| &mut cx.wasi)?; + wasmtime_wasi_http::add_to_linker(&mut linker, |cx: &mut Ctx| &mut cx.http)?; + + // Create our wasi context. + let builder = WasiCtxBuilder::new().inherit_stdio().arg(bin_name)?; + + let mut store = Store::new( + &engine, + Ctx { + wasi: builder.build(), + http: WasiHttp::new(), + }, + ); + + let instance = linker.instantiate(&mut store, &module)?; + let start = instance.get_typed_func::<(), ()>(&mut store, "_start")?; + start.call(&mut store, ()) +} diff --git a/crates/test-programs/tests/http_tests/utils.rs b/crates/test-programs/tests/http_tests/utils.rs new file mode 100644 index 000000000000..d455b96590c3 --- /dev/null +++ b/crates/test-programs/tests/http_tests/utils.rs @@ -0,0 +1,13 @@ +use std::path::Path; + +pub fn extract_exec_name_from_path(path: &Path) -> anyhow::Result { + path.file_stem() + .and_then(|s| s.to_str()) + .map(String::from) + .ok_or_else(|| { + anyhow::anyhow!( + "couldn't extract the file stem from path {}", + path.display() + ) + }) +} diff --git a/crates/test-programs/wasi-http-tests/Cargo.lock b/crates/test-programs/wasi-http-tests/Cargo.lock new file mode 100644 index 000000000000..b8d40a06d521 --- /dev/null +++ b/crates/test-programs/wasi-http-tests/Cargo.lock @@ -0,0 +1,380 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.9", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi-http-tests" +version = "0.0.0" +dependencies = [ + "anyhow", + "wit-bindgen", +] + +[[package]] +name = "wasm-encoder" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eff853c4f09eec94d76af527eddad4e9de13b11d6286a1ef7134bc30135a2b7" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6956efd8a1a2c48a707e9a1b2da729834a0f8e4c58117493b0d9d089cee468" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.102.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" +dependencies = [ + "indexmap", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7cf57f8786216c5652e1228b25203af2ff523808b5e9d3671894eee2bf7264" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef177b73007d86c720931d0e2ea7e30eb8c9776e58361717743fc1e83cfacfe5" +dependencies = [ + "anyhow", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdf5b00935b7b52d0e56cae1960f8ac13019a285f5aa762ff6bd7139a5c28a2" +dependencies = [ + "heck", + "wasm-metadata", + "wit-bindgen-core", + "wit-bindgen-rust-lib", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-lib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0a8f4b5fb1820b9d232beb122936425f72ec8fe6acb56e5d8782cfe55083da" +dependencies = [ + "heck", + "wit-bindgen-core", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadf1adf12ed25629b06272c16b335ef8c5a240d0ca64ab508a955ac3b46172c" +dependencies = [ + "anyhow", + "proc-macro2", + "syn 1.0.109", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "wit-component" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed04310239706efc71cc8b995cb0226089c5b5fd260c3bd800a71486bd3cec97" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "url", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f887c3da527a51b321076ebe6a7513026a4757b6d4d144259946552d6fc728b3" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "pulldown-cmark", + "unicode-xid", + "url", +] diff --git a/crates/test-programs/wasi-http-tests/Cargo.toml b/crates/test-programs/wasi-http-tests/Cargo.toml new file mode 100644 index 000000000000..0c04ce6987ba --- /dev/null +++ b/crates/test-programs/wasi-http-tests/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "wasi-http-tests" +version = "0.0.0" +readme = "README.md" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1" +wit-bindgen = "0.4.0" + +# This crate is built with the wasm32-wasi target, so it's separate +# from the main Wasmtime build, so use this directive to exclude it +# from the parent directory's workspace. +[workspace] diff --git a/crates/test-programs/wasi-http-tests/src/bin/outbound_request.rs b/crates/test-programs/wasi-http-tests/src/bin/outbound_request.rs new file mode 100644 index 000000000000..0e96aed29b68 --- /dev/null +++ b/crates/test-programs/wasi-http-tests/src/bin/outbound_request.rs @@ -0,0 +1,132 @@ +use anyhow::{anyhow, Context, Result}; +use std::fmt; +use wasi_http_tests::*; + +struct Response { + status: types::StatusCode, + headers: Vec<(String, String)>, + body: Vec, +} +impl fmt::Debug for Response { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut out = f.debug_struct("Response"); + out.field("status", &self.status) + .field("headers", &self.headers); + if let Ok(body) = std::str::from_utf8(&self.body) { + out.field("body", &body); + } else { + out.field("body", &self.body); + } + out.finish() + } +} + +fn request( + method: types::MethodParam<'_>, + scheme: types::SchemeParam<'_>, + authority: &str, + path: &str, + query: &str, + body: &[u8], +) -> Result { + let headers = types::new_fields(&[ + ("User-agent", "WASI-HTTP/0.0.1"), + ("Content-type", "application/json"), + ]); + + let request = + types::new_outgoing_request(method, path, query, Some(scheme), authority, headers); + + let request_body = types::outgoing_request_write(request) + .map_err(|_| anyhow!("outgoing request write failed"))?; + + let mut body_cursor = 0; + while body_cursor < body.len() { + let written = + streams::write(request_body, &body[body_cursor..]).context("writing request body")?; + body_cursor += written as usize; + } + + streams::drop_output_stream(request_body); + + let future_response = default_outgoing_http::handle(request, None); + // TODO: we could create a pollable from the future_response and poll on it here to test that + // its available immediately + + types::drop_outgoing_request(request); + + let incoming_response = types::future_incoming_response_get(future_response) + .ok_or_else(|| anyhow!("incoming response is available immediately"))? + // TODO: maybe anything that appears in the Result<_, E> position should impl + // Error? anyway, just use its Debug here: + .map_err(|e| anyhow!("incoming response error: {e:?}"))?; + + types::drop_future_incoming_response(future_response); + + let status = types::incoming_response_status(incoming_response); + + let headers_handle = types::incoming_response_headers(incoming_response); + let headers = types::fields_entries(headers_handle); + types::drop_fields(headers_handle); + + let body_stream = types::incoming_response_consume(incoming_response) + .map_err(|()| anyhow!("incoming response has no body stream"))?; + types::drop_incoming_response(incoming_response); + + let mut body = Vec::new(); + let mut eof = false; + while !eof { + let (mut body_chunk, stream_ended) = streams::read(body_stream, u64::MAX)?; + eof = stream_ended; + body.append(&mut body_chunk); + } + streams::drop_input_stream(body_stream); + + Ok(Response { + status, + headers, + body, + }) +} + +fn main() -> Result<()> { + let r1 = request( + types::MethodParam::Get, + types::SchemeParam::Http, + "postman-echo.com", + "/get", + "?some=arg?goes=here", + &[], + ) + .context("postman-echo /get")?; + + println!("postman-echo /get: {r1:?}"); + assert_eq!(r1.status, 200); + + let r2 = request( + types::MethodParam::Post, + types::SchemeParam::Http, + "postman-echo.com", + "/post", + "", + b"{\"foo\": \"bar\"}", + ) + .context("postman-echo /post")?; + + println!("postman-echo /post: {r2:?}"); + assert_eq!(r2.status, 200); + + let r3 = request( + types::MethodParam::Put, + types::SchemeParam::Http, + "postman-echo.com", + "/put", + "", + &[], + ) + .context("postman-echo /put")?; + + println!("postman-echo /put: {r3:?}"); + assert_eq!(r3.status, 200); + Ok(()) +} diff --git a/crates/test-programs/wasi-http-tests/src/lib.rs b/crates/test-programs/wasi-http-tests/src/lib.rs new file mode 100644 index 000000000000..7da3c0508914 --- /dev/null +++ b/crates/test-programs/wasi-http-tests/src/lib.rs @@ -0,0 +1,3 @@ +// The macro will generate a macro for defining exports which we won't be reusing +#![allow(unused)] +wit_bindgen::generate!({ path: "../../wasi-http/wasi-http/wit" }); diff --git a/crates/wasi-http/Cargo.toml b/crates/wasi-http/Cargo.toml new file mode 100644 index 000000000000..e5b88e9f6ff3 --- /dev/null +++ b/crates/wasi-http/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "wasmtime-wasi-http" +version = "0.0.1" +authors.workspace = true +edition.workspace = true +repository = "https://github.com/bytecodealliance/wasmtime" +license = "Apache-2.0 WITH LLVM-exception" +description = "Experimental HTTP library for WebAssembly in Wasmtime" +readme = "readme.md" + +[dependencies] +anyhow = { workspace = true } +bytes = "1.1.0" +hyper = { version = "1.0.0-rc.3", features = ["full"] } +tokio = { version = "1", default-features = false, features = ["net", "rt-multi-thread", "time"] } +http = { version = "0.2.9" } +http-body = "1.0.0-rc.2" +http-body-util = "0.1.0-rc.2" +thiserror = { workspace = true } +wasmtime = { workspace = true, features = ['component-model'] } + +# The `ring` crate, used to implement TLS, does not build on riscv64 or s390x +[target.'cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))'.dependencies] +tokio-rustls = { version = "0.24.0" } +rustls = { version = "0.21.0" } +webpki-roots = { version = "0.23.0" } diff --git a/crates/wasi-http/src/component_impl.rs b/crates/wasi-http/src/component_impl.rs new file mode 100644 index 000000000000..a77af025e53e --- /dev/null +++ b/crates/wasi-http/src/component_impl.rs @@ -0,0 +1,431 @@ +use crate::default_outgoing_http::Host; +pub use crate::r#struct::WasiHttp; +use crate::streams::Host as StreamsHost; +use crate::types::{Host as TypesHost, RequestOptions, Scheme}; +use anyhow::anyhow; +use std::str; +use std::vec::Vec; +use wasmtime::{AsContext, AsContextMut, Caller, Extern, Memory}; + +const MEMORY: &str = "memory"; + +#[derive(Debug, thiserror::Error)] +enum HttpError { + #[error("Memory not found")] + MemoryNotFound, + #[error("Memory access error")] + MemoryAccessError(#[from] wasmtime::MemoryAccessError), + #[error("Buffer too small")] + BufferTooSmall, + #[error("UTF-8 error")] + Utf8Error(#[from] std::str::Utf8Error), +} + +fn memory_get(caller: &mut Caller<'_, T>) -> Result { + if let Some(Extern::Memory(mem)) = caller.get_export(MEMORY) { + Ok(mem) + } else { + Err(HttpError::MemoryNotFound) + } +} + +/// Get a slice of length `len` from `memory`, starting at `offset`. +/// This will return an `HttpError::BufferTooSmall` if the size of the +/// requested slice is larger than the memory size. +fn slice_from_memory( + memory: &Memory, + mut ctx: impl AsContextMut, + offset: u32, + len: u32, +) -> Result, HttpError> { + let required_memory_size = offset.checked_add(len).ok_or(HttpError::BufferTooSmall)? as usize; + + if required_memory_size > memory.data_size(&mut ctx) { + return Err(HttpError::BufferTooSmall); + } + + let mut buf = vec![0u8; len as usize]; + memory.read(&mut ctx, offset as usize, buf.as_mut_slice())?; + Ok(buf) +} + +fn u32_from_memory(memory: &Memory, ctx: impl AsContextMut, ptr: u32) -> Result { + let slice = slice_from_memory(memory, ctx, ptr, 4)?; + let mut dst = [0u8; 4]; + dst.clone_from_slice(&slice[0..4]); + Ok(u32::from_le_bytes(dst)) +} + +/// Read a string of byte length `len` from `memory`, starting at `offset`. +fn string_from_memory( + memory: &Memory, + ctx: impl AsContextMut, + offset: u32, + len: u32, +) -> Result { + let slice = slice_from_memory(memory, ctx, offset, len)?; + Ok(std::str::from_utf8(&slice)?.to_string()) +} + +fn allocate_guest_pointer(caller: &mut Caller<'_, T>, size: u32) -> anyhow::Result { + let realloc = caller + .get_export("cabi_realloc") + .ok_or_else(|| anyhow!("missing required export cabi_realloc"))?; + let func = realloc + .into_func() + .ok_or_else(|| anyhow!("cabi_realloc must be a func"))?; + let typed = func.typed::<(u32, u32, u32, u32), u32>(caller.as_context())?; + Ok(typed.call(caller.as_context_mut(), (0, 0, 4, size))?) +} + +fn u32_array_to_u8(arr: &[u32]) -> Vec { + let mut result = std::vec::Vec::new(); + for val in arr.iter() { + let bytes = val.to_le_bytes(); + for b in bytes.iter() { + result.push(*b); + } + } + result +} + +pub fn add_component_to_linker( + linker: &mut wasmtime::Linker, + get_cx: impl Fn(&mut T) -> &mut WasiHttp + Send + Sync + Copy + 'static, +) -> anyhow::Result<()> { + linker.func_wrap( + "default-outgoing-HTTP", + "handle", + move |mut caller: Caller<'_, T>, + request: u32, + has_options: i32, + has_timeout: i32, + timeout_ms: u32, + has_first_byte_timeout: i32, + first_byte_timeout_ms: u32, + has_between_bytes_timeout: i32, + between_bytes_timeout_ms: u32| + -> anyhow::Result { + let options = if has_options == 1 { + Some(RequestOptions { + connect_timeout_ms: if has_timeout == 1 { + Some(timeout_ms) + } else { + None + }, + first_byte_timeout_ms: if has_first_byte_timeout == 1 { + Some(first_byte_timeout_ms) + } else { + None + }, + between_bytes_timeout_ms: if has_between_bytes_timeout == 1 { + Some(between_bytes_timeout_ms) + } else { + None + }, + }) + } else { + None + }; + + Ok(get_cx(caller.data_mut()).handle(request, options)?) + }, + )?; + linker.func_wrap( + "types", + "new-outgoing-request", + move |mut caller: Caller<'_, T>, + method: i32, + _b: i32, + _c: i32, + path_ptr: u32, + path_len: u32, + query_ptr: u32, + query_len: u32, + scheme_is_some: i32, + scheme: i32, + _h: i32, + _i: i32, + authority_ptr: u32, + authority_len: u32, + headers: u32| + -> anyhow::Result { + let memory = memory_get(&mut caller)?; + let path = string_from_memory(&memory, caller.as_context_mut(), path_ptr, path_len)?; + let query = string_from_memory(&memory, caller.as_context_mut(), query_ptr, query_len)?; + let authority = string_from_memory( + &memory, + caller.as_context_mut(), + authority_ptr, + authority_len, + )?; + + let mut s = Scheme::Https; + if scheme_is_some == 1 { + s = match scheme { + 0 => Scheme::Http, + 1 => Scheme::Https, + _ => anyhow::bail!("unsupported scheme {scheme}"), + }; + } + let m = match method { + 0 => crate::types::Method::Get, + 1 => crate::types::Method::Head, + 2 => crate::types::Method::Post, + 3 => crate::types::Method::Put, + 4 => crate::types::Method::Delete, + 5 => crate::types::Method::Connect, + 6 => crate::types::Method::Options, + 7 => crate::types::Method::Trace, + 8 => crate::types::Method::Patch, + _ => anyhow::bail!("unsupported method {method}"), + }; + + let ctx = get_cx(caller.data_mut()); + Ok(ctx.new_outgoing_request(m, path, query, Some(s), authority, headers)?) + }, + )?; + linker.func_wrap( + "types", + "incoming-response-status", + move |mut caller: Caller<'_, T>, id: u32| -> anyhow::Result { + let ctx = get_cx(caller.data_mut()); + Ok(ctx.incoming_response_status(id)?.into()) + }, + )?; + linker.func_wrap( + "types", + "drop-future-incoming-response", + move |_caller: Caller<'_, T>, _future: u32| -> anyhow::Result<()> { + // FIXME: Intentionally left blank + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "future-incoming-response-get", + move |mut caller: Caller<'_, T>, future: u32, ptr: i32| -> anyhow::Result<()> { + let memory = memory_get(&mut caller)?; + + // First == is_some + // Second == is_err + // Third == {ok: is_err = false, tag: is_err = true} + // Fourth == string ptr + // Fifth == string len + let result: [u32; 5] = [1, 0, future, 0, 0]; + let raw = u32_array_to_u8(&result); + + memory.write(caller.as_context_mut(), ptr as _, &raw)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "incoming-response-consume", + move |mut caller: Caller<'_, T>, response: u32, ptr: i32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + let stream = ctx.incoming_response_consume(response)?.unwrap_or(0); + + let memory = memory_get(&mut caller).unwrap(); + + // First == is_some + // Second == stream_id + let result: [u32; 2] = [0, stream]; + let raw = u32_array_to_u8(&result); + + memory.write(caller.as_context_mut(), ptr as _, &raw)?; + Ok(()) + }, + )?; + linker.func_wrap( + "poll", + "drop-pollable", + move |_caller: Caller<'_, T>, _a: i32| -> anyhow::Result<()> { + anyhow::bail!("unimplemented") + }, + )?; + linker.func_wrap( + "types", + "drop-fields", + move |mut caller: Caller<'_, T>, ptr: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + ctx.drop_fields(ptr)?; + Ok(()) + }, + )?; + linker.func_wrap( + "streams", + "drop-input-stream", + move |mut caller: Caller<'_, T>, id: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + ctx.drop_input_stream(id)?; + Ok(()) + }, + )?; + linker.func_wrap( + "streams", + "drop-output-stream", + move |mut caller: Caller<'_, T>, id: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + ctx.drop_output_stream(id)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "outgoing-request-write", + move |mut caller: Caller<'_, T>, request: u32, ptr: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + let stream = ctx + .outgoing_request_write(request)? + .map_err(|_| anyhow!("no outgoing stream present"))?; + + let memory = memory_get(&mut caller)?; + // First == is_some + // Second == stream_id + let result: [u32; 2] = [0, stream]; + let raw = u32_array_to_u8(&result); + + memory.write(caller.as_context_mut(), ptr as _, &raw)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "drop-outgoing-request", + move |mut caller: Caller<'_, T>, id: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + ctx.drop_outgoing_request(id)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "drop-incoming-response", + move |mut caller: Caller<'_, T>, id: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + ctx.drop_incoming_response(id)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "new-fields", + move |mut caller: Caller<'_, T>, base_ptr: u32, len: u32| -> anyhow::Result { + let memory = memory_get(&mut caller)?; + + let mut vec = Vec::new(); + let mut i = 0; + // TODO: read this more efficiently as a single block. + while i < len { + let ptr = base_ptr + i * 16; + let name_ptr = u32_from_memory(&memory, caller.as_context_mut(), ptr)?; + let name_len = u32_from_memory(&memory, caller.as_context_mut(), ptr + 4)?; + let value_ptr = u32_from_memory(&memory, caller.as_context_mut(), ptr + 8)?; + let value_len = u32_from_memory(&memory, caller.as_context_mut(), ptr + 12)?; + + let name = + string_from_memory(&memory, caller.as_context_mut(), name_ptr, name_len)?; + let value = + string_from_memory(&memory, caller.as_context_mut(), value_ptr, value_len)?; + + vec.push((name, value)); + i = i + 1; + } + + let ctx = get_cx(caller.data_mut()); + Ok(ctx.new_fields(vec)?) + }, + )?; + linker.func_wrap( + "streams", + "read", + move |mut caller: Caller<'_, T>, stream: u32, len: u64, ptr: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + let bytes_tuple = ctx.read(stream, len)??; + let bytes = bytes_tuple.0; + let done = match bytes_tuple.1 { + true => 1, + false => 0, + }; + let body_len: u32 = bytes.len().try_into()?; + let out_ptr = allocate_guest_pointer(&mut caller, body_len)?; + let result: [u32; 4] = [0, out_ptr, body_len, done]; + let raw = u32_array_to_u8(&result); + + let memory = memory_get(&mut caller)?; + memory.write(caller.as_context_mut(), out_ptr as _, &bytes)?; + memory.write(caller.as_context_mut(), ptr as _, &raw)?; + Ok(()) + }, + )?; + linker.func_wrap( + "streams", + "write", + move |mut caller: Caller<'_, T>, + stream: u32, + body_ptr: u32, + body_len: u32, + ptr: u32| + -> anyhow::Result<()> { + let memory = memory_get(&mut caller)?; + let body = string_from_memory(&memory, caller.as_context_mut(), body_ptr, body_len)?; + + let result: [u32; 3] = [0, 0, body_len]; + let raw = u32_array_to_u8(&result); + + let memory = memory_get(&mut caller)?; + memory.write(caller.as_context_mut(), ptr as _, &raw)?; + + let ctx = get_cx(caller.data_mut()); + ctx.write(stream, body.as_bytes().to_vec())??; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "fields-entries", + move |mut caller: Caller<'_, T>, fields: u32, out_ptr: u32| -> anyhow::Result<()> { + let ctx = get_cx(caller.data_mut()); + let entries = ctx.fields_entries(fields)?; + + let header_len = entries.len(); + let tuple_ptr = allocate_guest_pointer(&mut caller, (16 * header_len).try_into()?)?; + let mut ptr = tuple_ptr; + for item in entries.iter() { + let name = &item.0; + let value = &item.1; + let name_len: u32 = name.len().try_into()?; + let value_len: u32 = value.len().try_into()?; + + let name_ptr = allocate_guest_pointer(&mut caller, name_len)?; + let value_ptr = allocate_guest_pointer(&mut caller, value_len)?; + + let memory = memory_get(&mut caller)?; + memory.write(caller.as_context_mut(), name_ptr as _, &name.as_bytes())?; + memory.write(caller.as_context_mut(), value_ptr as _, &value.as_bytes())?; + + let pair: [u32; 4] = [name_ptr, name_len, value_ptr, value_len]; + let raw_pair = u32_array_to_u8(&pair); + memory.write(caller.as_context_mut(), ptr as _, &raw_pair)?; + + ptr = ptr + 16; + } + + let memory = memory_get(&mut caller)?; + let result: [u32; 2] = [tuple_ptr, header_len.try_into()?]; + let raw = u32_array_to_u8(&result); + memory.write(caller.as_context_mut(), out_ptr as _, &raw)?; + Ok(()) + }, + )?; + linker.func_wrap( + "types", + "incoming-response-headers", + move |mut caller: Caller<'_, T>, handle: u32| -> anyhow::Result { + let ctx = get_cx(caller.data_mut()); + Ok(ctx.incoming_response_headers(handle)?) + }, + )?; + Ok(()) +} diff --git a/crates/wasi-http/src/http_impl.rs b/crates/wasi-http/src/http_impl.rs new file mode 100644 index 000000000000..206ebabcc0c3 --- /dev/null +++ b/crates/wasi-http/src/http_impl.rs @@ -0,0 +1,211 @@ +use crate::r#struct::ActiveResponse; +pub use crate::r#struct::WasiHttp; +use crate::types::{RequestOptions, Scheme}; +#[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] +use anyhow::anyhow; +use anyhow::bail; +use bytes::{BufMut, Bytes, BytesMut}; +use http_body_util::{BodyExt, Full}; +use hyper::Method; +use hyper::Request; +#[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpStream; +use tokio::runtime::Runtime; +use tokio::time::timeout; +#[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] +use tokio_rustls::rustls::{self, OwnedTrustAnchor}; + +impl crate::default_outgoing_http::Host for WasiHttp { + fn handle( + &mut self, + request_id: crate::default_outgoing_http::OutgoingRequest, + options: Option, + ) -> wasmtime::Result { + // TODO: Initialize this once? + let rt = Runtime::new().unwrap(); + let _enter = rt.enter(); + + let f = self.handle_async(request_id, options); + match rt.block_on(f) { + Ok(r) => { + println!("{} OK", r); + Ok(r) + } + Err(e) => { + println!("{} ERR", e); + Err(e) + } + } + } +} + +fn port_for_scheme(scheme: &Option) -> &str { + match scheme { + Some(s) => match s { + Scheme::Http => ":80", + Scheme::Https => ":443", + // This should never happen. + _ => panic!("unsupported scheme!"), + }, + None => ":443", + } +} + +impl WasiHttp { + async fn handle_async( + &mut self, + request_id: crate::default_outgoing_http::OutgoingRequest, + options: Option, + ) -> wasmtime::Result { + let opts = options.unwrap_or( + // TODO: Configurable defaults here? + RequestOptions { + connect_timeout_ms: Some(600 * 1000), + first_byte_timeout_ms: Some(600 * 1000), + between_bytes_timeout_ms: Some(600 * 1000), + }, + ); + let connect_timeout = + Duration::from_millis(opts.connect_timeout_ms.unwrap_or(600 * 1000).into()); + let first_bytes_timeout = + Duration::from_millis(opts.first_byte_timeout_ms.unwrap_or(600 * 1000).into()); + let between_bytes_timeout = + Duration::from_millis(opts.between_bytes_timeout_ms.unwrap_or(600 * 1000).into()); + + let request = match self.requests.get(&request_id) { + Some(r) => r, + None => bail!("not found!"), + }; + + let method = match request.method { + crate::types::Method::Get => Method::GET, + crate::types::Method::Head => Method::HEAD, + crate::types::Method::Post => Method::POST, + crate::types::Method::Put => Method::PUT, + crate::types::Method::Delete => Method::DELETE, + crate::types::Method::Connect => Method::CONNECT, + crate::types::Method::Options => Method::OPTIONS, + crate::types::Method::Trace => Method::TRACE, + crate::types::Method::Patch => Method::PATCH, + _ => bail!("unknown method!"), + }; + + let scheme = match request.scheme.as_ref().unwrap_or(&Scheme::Https) { + Scheme::Http => "http://", + Scheme::Https => "https://", + // TODO: this is wrong, fix this. + _ => panic!("Unsupported scheme!"), + }; + + // Largely adapted from https://hyper.rs/guides/1/client/basic/ + let authority = match request.authority.find(":") { + Some(_) => request.authority.clone(), + None => request.authority.clone() + port_for_scheme(&request.scheme), + }; + let mut sender = if scheme == "https://" { + #[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] + { + let stream = TcpStream::connect(authority.clone()).await?; + //TODO: uncomment this code and make the tls implementation a feature decision. + //let connector = tokio_native_tls::native_tls::TlsConnector::builder().build()?; + //let connector = tokio_native_tls::TlsConnector::from(connector); + //let host = authority.split(":").next().unwrap_or(&authority); + //let stream = connector.connect(&host, stream).await?; + + // derived from https://github.com/tokio-rs/tls/blob/master/tokio-rustls/examples/client/src/main.rs + let mut root_cert_store = rustls::RootCertStore::empty(); + root_cert_store.add_server_trust_anchors( + webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| { + OwnedTrustAnchor::from_subject_spki_name_constraints( + ta.subject, + ta.spki, + ta.name_constraints, + ) + }), + ); + let config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(root_cert_store) + .with_no_client_auth(); + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); + let mut parts = authority.split(":"); + let host = parts.next().unwrap_or(&authority); + let domain = + rustls::ServerName::try_from(host).map_err(|_| anyhow!("invalid dnsname"))?; + let stream = connector.connect(domain, stream).await?; + + let t = timeout( + connect_timeout, + hyper::client::conn::http1::handshake(stream), + ) + .await?; + let (s, conn) = t?; + tokio::task::spawn(async move { + if let Err(err) = conn.await { + println!("Connection failed: {:?}", err); + } + }); + s + } + #[cfg(any(target_arch = "riscv64", target_arch = "s390x"))] + bail!("unsupported architecture for SSL") + } else { + let tcp = TcpStream::connect(authority).await?; + let t = timeout(connect_timeout, hyper::client::conn::http1::handshake(tcp)).await?; + let (s, conn) = t?; + tokio::task::spawn(async move { + if let Err(err) = conn.await { + println!("Connection failed: {:?}", err); + } + }); + s + }; + + let url = scheme.to_owned() + &request.authority + &request.path + &request.query; + + let mut call = Request::builder() + .method(method) + .uri(url) + .header(hyper::header::HOST, request.authority.as_str()); + + for (key, val) in request.headers.iter() { + for item in val { + call = call.header(key, item.clone()); + } + } + + let response_id = self.response_id_base; + self.response_id_base = self.response_id_base + 1; + let mut response = ActiveResponse::new(response_id); + let body = Full::::new( + self.streams + .get(&request.body) + .unwrap_or(&Bytes::new()) + .clone(), + ); + let t = timeout(first_bytes_timeout, sender.send_request(call.body(body)?)).await?; + let mut res = t?; + response.status = res.status().try_into()?; + for (key, value) in res.headers().iter() { + let mut vec = std::vec::Vec::new(); + vec.push(value.to_str()?.to_string()); + response + .response_headers + .insert(key.as_str().to_string(), vec); + } + let mut buf = BytesMut::new(); + while let Some(next) = timeout(between_bytes_timeout, res.frame()).await? { + let frame = next?; + if let Some(chunk) = frame.data_ref() { + buf.put(chunk.clone()); + } + } + response.body = self.streams_id_base; + self.streams_id_base = self.streams_id_base + 1; + self.streams.insert(response.body, buf.freeze()); + self.responses.insert(response_id, response); + Ok(response_id) + } +} diff --git a/crates/wasi-http/src/lib.rs b/crates/wasi-http/src/lib.rs new file mode 100644 index 000000000000..4f90015cc5ce --- /dev/null +++ b/crates/wasi-http/src/lib.rs @@ -0,0 +1,27 @@ +use crate::component_impl::add_component_to_linker; +pub use crate::r#struct::WasiHttp; + +wasmtime::component::bindgen!({ path: "wasi-http/wit", world: "proxy"}); + +pub mod component_impl; +pub mod http_impl; +pub mod streams_impl; +pub mod r#struct; +pub mod types_impl; + +pub fn add_to_component_linker( + linker: &mut wasmtime::component::Linker, + get_cx: impl Fn(&mut T) -> &mut WasiHttp + Send + Sync + Copy + 'static, +) -> anyhow::Result<()> { + default_outgoing_http::add_to_linker(linker, get_cx)?; + types::add_to_linker(linker, get_cx)?; + streams::add_to_linker(linker, get_cx)?; + Ok(()) +} + +pub fn add_to_linker( + linker: &mut wasmtime::Linker, + get_cx: impl Fn(&mut T) -> &mut WasiHttp + Send + Sync + Copy + 'static, +) -> anyhow::Result<()> { + add_component_to_linker(linker, get_cx) +} diff --git a/crates/wasi-http/src/streams_impl.rs b/crates/wasi-http/src/streams_impl.rs new file mode 100644 index 000000000000..afaf0153e655 --- /dev/null +++ b/crates/wasi-http/src/streams_impl.rs @@ -0,0 +1,93 @@ +use crate::poll::Pollable; +use crate::streams::{InputStream, OutputStream, StreamError}; +use crate::WasiHttp; +use anyhow::{anyhow, bail}; +use std::vec::Vec; + +impl crate::streams::Host for WasiHttp { + fn read( + &mut self, + stream: InputStream, + len: u64, + ) -> wasmtime::Result, bool), StreamError>> { + let s = self + .streams + .get_mut(&stream) + .ok_or_else(|| anyhow!("stream not found: {stream}"))?; + if len == 0 { + Ok(Ok((bytes::Bytes::new().to_vec(), s.len() > 0))) + } else if s.len() > len.try_into()? { + let result = s.split_to(len.try_into()?); + Ok(Ok((result.to_vec(), false))) + } else { + s.truncate(s.len()); + Ok(Ok((s.clone().to_vec(), true))) + } + } + + fn skip( + &mut self, + _this: InputStream, + _len: u64, + ) -> wasmtime::Result> { + bail!("unimplemented: skip"); + } + + fn subscribe_to_input_stream(&mut self, _this: InputStream) -> wasmtime::Result { + bail!("unimplemented: subscribe_to_input_stream"); + } + + fn drop_input_stream(&mut self, stream: InputStream) -> wasmtime::Result<()> { + let r = self + .streams + .get_mut(&stream) + .ok_or_else(|| anyhow!("no such input-stream {stream}"))?; + r.truncate(0); + Ok(()) + } + + fn write( + &mut self, + this: OutputStream, + buf: Vec, + ) -> wasmtime::Result> { + // TODO: Make this a real write not a replace. + self.streams.insert(this, bytes::Bytes::from(buf.clone())); + Ok(Ok(buf.len().try_into()?)) + } + + fn write_zeroes( + &mut self, + _this: OutputStream, + _len: u64, + ) -> wasmtime::Result> { + bail!("unimplemented: write_zeroes"); + } + + fn splice( + &mut self, + _this: OutputStream, + _src: InputStream, + _len: u64, + ) -> wasmtime::Result> { + bail!("unimplemented: splice"); + } + + fn forward( + &mut self, + _this: OutputStream, + _src: InputStream, + ) -> wasmtime::Result> { + bail!("unimplemented: forward"); + } + + fn subscribe_to_output_stream(&mut self, _this: OutputStream) -> wasmtime::Result { + bail!("unimplemented: subscribe_to_output_stream"); + } + + fn drop_output_stream(&mut self, _this: OutputStream) -> wasmtime::Result<()> { + //bail!("unimplemented: drop_output_stream"); + //FIXME: intentionally ignoring + Ok(()) + } +} diff --git a/crates/wasi-http/src/struct.rs b/crates/wasi-http/src/struct.rs new file mode 100644 index 000000000000..b1c02ad1a21e --- /dev/null +++ b/crates/wasi-http/src/struct.rs @@ -0,0 +1,80 @@ +use crate::types::{Method, Scheme}; +use bytes::Bytes; +use std::collections::HashMap; + +#[derive(Clone)] +pub struct WasiHttp { + pub request_id_base: u32, + pub response_id_base: u32, + pub fields_id_base: u32, + pub streams_id_base: u32, + pub requests: HashMap, + pub responses: HashMap, + pub fields: HashMap>>, + pub streams: HashMap, +} + +#[derive(Clone)] +pub struct ActiveRequest { + pub id: u32, + pub active_request: bool, + pub method: Method, + pub scheme: Option, + pub path: String, + pub query: String, + pub authority: String, + pub headers: HashMap>, + pub body: u32, +} + +#[derive(Clone)] +pub struct ActiveResponse { + pub id: u32, + pub active_response: bool, + pub status: u16, + pub body: u32, + pub response_headers: HashMap>, +} + +impl ActiveRequest { + pub fn new(id: u32) -> Self { + Self { + id: id, + active_request: false, + method: Method::Get, + scheme: Some(Scheme::Http), + path: "".to_string(), + query: "".to_string(), + authority: "".to_string(), + headers: HashMap::new(), + body: 0, + } + } +} + +impl ActiveResponse { + pub fn new(id: u32) -> Self { + Self { + id: id, + active_response: false, + status: 0, + body: 0, + response_headers: HashMap::new(), + } + } +} + +impl WasiHttp { + pub fn new() -> Self { + Self { + request_id_base: 1, + response_id_base: 1, + fields_id_base: 1, + streams_id_base: 1, + requests: HashMap::new(), + responses: HashMap::new(), + fields: HashMap::new(), + streams: HashMap::new(), + } + } +} diff --git a/crates/wasi-http/src/types_impl.rs b/crates/wasi-http/src/types_impl.rs new file mode 100644 index 000000000000..61e5d3b1ae0f --- /dev/null +++ b/crates/wasi-http/src/types_impl.rs @@ -0,0 +1,271 @@ +use crate::poll::Pollable; +use crate::r#struct::ActiveRequest; +use crate::types::{ + Error, Fields, FutureIncomingResponse, Headers, IncomingRequest, IncomingResponse, + IncomingStream, Method, OutgoingRequest, OutgoingResponse, OutgoingStream, ResponseOutparam, + Scheme, StatusCode, Trailers, +}; +use crate::WasiHttp; +use anyhow::{anyhow, bail}; +use std::collections::HashMap; + +impl crate::types::Host for WasiHttp { + fn drop_fields(&mut self, fields: Fields) -> wasmtime::Result<()> { + self.fields.remove(&fields); + Ok(()) + } + fn new_fields(&mut self, entries: Vec<(String, String)>) -> wasmtime::Result { + let mut map = HashMap::new(); + for item in entries.iter() { + let mut vec = std::vec::Vec::new(); + vec.push(item.1.clone()); + map.insert(item.0.clone(), vec); + } + + let id = self.fields_id_base; + self.fields_id_base = id + 1; + self.fields.insert(id, map); + + Ok(id) + } + fn fields_get(&mut self, fields: Fields, name: String) -> wasmtime::Result> { + let res = self + .fields + .get(&fields) + .ok_or_else(|| anyhow!("fields not found: {fields}"))? + .get(&name) + .ok_or_else(|| anyhow!("key not found: {name}"))? + .clone(); + Ok(res) + } + fn fields_set( + &mut self, + fields: Fields, + name: String, + value: Vec, + ) -> wasmtime::Result<()> { + match self.fields.get_mut(&fields) { + Some(m) => { + m.insert(name, value.clone()); + Ok(()) + } + None => bail!("fields not found"), + } + } + fn fields_delete(&mut self, fields: Fields, name: String) -> wasmtime::Result<()> { + match self.fields.get_mut(&fields) { + Some(m) => m.remove(&name), + None => None, + }; + Ok(()) + } + fn fields_append( + &mut self, + fields: Fields, + name: String, + value: String, + ) -> wasmtime::Result<()> { + let m = self + .fields + .get_mut(&fields) + .ok_or_else(|| anyhow!("unknown fields: {fields}"))?; + match m.get_mut(&name) { + Some(v) => v.push(value), + None => { + let mut vec = std::vec::Vec::new(); + vec.push(value); + m.insert(name, vec); + } + }; + Ok(()) + } + fn fields_entries(&mut self, fields: Fields) -> wasmtime::Result> { + let field_map = match self.fields.get(&fields) { + Some(m) => m, + None => bail!("fields not found."), + }; + let mut result = Vec::new(); + for (name, value) in field_map { + result.push((name.clone(), value[0].clone())); + } + Ok(result) + } + fn fields_clone(&mut self, fields: Fields) -> wasmtime::Result { + let id = self.fields_id_base; + self.fields_id_base = self.fields_id_base + 1; + + let m = self + .fields + .get(&fields) + .ok_or_else(|| anyhow!("fields not found: {fields}"))?; + self.fields.insert(id, m.clone()); + Ok(id) + } + fn finish_incoming_stream(&mut self, _s: IncomingStream) -> wasmtime::Result> { + bail!("unimplemented: finish_incoming_stream") + } + fn finish_outgoing_stream( + &mut self, + _s: OutgoingStream, + _trailers: Option, + ) -> wasmtime::Result<()> { + bail!("unimplemented: finish_outgoing_stream") + } + fn drop_incoming_request(&mut self, _request: IncomingRequest) -> wasmtime::Result<()> { + bail!("unimplemented: drop_incoming_request") + } + fn drop_outgoing_request(&mut self, request: OutgoingRequest) -> wasmtime::Result<()> { + self.requests.remove(&request); + Ok(()) + } + fn incoming_request_method(&mut self, _request: IncomingRequest) -> wasmtime::Result { + bail!("unimplemented: incoming_request_method") + } + fn incoming_request_path(&mut self, _request: IncomingRequest) -> wasmtime::Result { + bail!("unimplemented: incoming_request_path") + } + fn incoming_request_scheme( + &mut self, + _request: IncomingRequest, + ) -> wasmtime::Result> { + bail!("unimplemented: incoming_request_scheme") + } + fn incoming_request_authority( + &mut self, + _request: IncomingRequest, + ) -> wasmtime::Result { + bail!("unimplemented: incoming_request_authority") + } + fn incoming_request_headers(&mut self, _request: IncomingRequest) -> wasmtime::Result { + bail!("unimplemented: incoming_request_headers") + } + fn incoming_request_consume( + &mut self, + _request: IncomingRequest, + ) -> wasmtime::Result> { + bail!("unimplemented: incoming_request_consume") + } + fn incoming_request_query(&mut self, _request: IncomingRequest) -> wasmtime::Result { + bail!("unimplemented: incoming_request_query") + } + fn new_outgoing_request( + &mut self, + method: Method, + path: String, + query: String, + scheme: Option, + authority: String, + headers: Headers, + ) -> wasmtime::Result { + let id = self.request_id_base; + self.request_id_base = self.request_id_base + 1; + + let mut req = ActiveRequest::new(id); + req.path = path; + req.query = query; + req.authority = authority; + req.method = method; + req.headers = match self.fields.get(&headers) { + Some(h) => h.clone(), + None => bail!("headers not found."), + }; + req.scheme = scheme; + self.requests.insert(id, req); + Ok(id) + } + fn outgoing_request_write( + &mut self, + request: OutgoingRequest, + ) -> wasmtime::Result> { + let mut req = self + .requests + .get_mut(&request) + .ok_or_else(|| anyhow!("unknown request: {request}"))?; + req.body = self.streams_id_base; + self.streams_id_base = self.streams_id_base + 1; + Ok(Ok(req.body)) + } + fn drop_response_outparam(&mut self, _response: ResponseOutparam) -> wasmtime::Result<()> { + bail!("unimplemented: drop_response_outparam") + } + fn set_response_outparam( + &mut self, + _outparam: ResponseOutparam, + _response: Result, + ) -> wasmtime::Result> { + bail!("unimplemented: set_response_outparam") + } + fn drop_incoming_response(&mut self, response: IncomingResponse) -> wasmtime::Result<()> { + self.responses.remove(&response); + Ok(()) + } + fn drop_outgoing_response(&mut self, _response: OutgoingResponse) -> wasmtime::Result<()> { + bail!("unimplemented: drop_outgoing_response") + } + fn incoming_response_status( + &mut self, + response: IncomingResponse, + ) -> wasmtime::Result { + let r = self + .responses + .get(&response) + .ok_or_else(|| anyhow!("response not found: {response}"))?; + Ok(r.status) + } + fn incoming_response_headers( + &mut self, + response: IncomingResponse, + ) -> wasmtime::Result { + let r = self + .responses + .get(&response) + .ok_or_else(|| anyhow!("response not found: {response}"))?; + let id = self.fields_id_base; + self.fields_id_base = self.fields_id_base + 1; + + self.fields.insert(id, r.response_headers.clone()); + Ok(id) + } + fn incoming_response_consume( + &mut self, + response: IncomingResponse, + ) -> wasmtime::Result> { + let r = self + .responses + .get(&response) + .ok_or_else(|| anyhow!("response not found: {response}"))?; + + Ok(Ok(r.body)) + } + fn new_outgoing_response( + &mut self, + _status_code: StatusCode, + _headers: Headers, + ) -> wasmtime::Result { + bail!("unimplemented: new_outgoing_response") + } + fn outgoing_response_write( + &mut self, + _response: OutgoingResponse, + ) -> wasmtime::Result> { + bail!("unimplemented: outgoing_response_write") + } + fn drop_future_incoming_response( + &mut self, + _f: FutureIncomingResponse, + ) -> wasmtime::Result<()> { + bail!("unimplemented: drop_future_incoming_response") + } + fn future_incoming_response_get( + &mut self, + _f: FutureIncomingResponse, + ) -> wasmtime::Result>> { + bail!("unimplemented: future_incoming_response_get") + } + fn listen_to_future_incoming_response( + &mut self, + _f: FutureIncomingResponse, + ) -> wasmtime::Result { + bail!("unimplemented: listen_to_future_incoming_response") + } +} diff --git a/crates/wasi-http/wasi-http b/crates/wasi-http/wasi-http new file mode 160000 index 000000000000..244e068c2de4 --- /dev/null +++ b/crates/wasi-http/wasi-http @@ -0,0 +1 @@ +Subproject commit 244e068c2de43088bda308fcdf51ed2479d885f5 diff --git a/scripts/publish.rs b/scripts/publish.rs index 786d352fd866..bbf54c3fab9e 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -68,6 +68,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ // other misc wasmtime crates "wasmtime-wasi", "wasmtime-wasi-crypto", + "wasmtime-wasi-http", "wasmtime-wasi-nn", "wasmtime-wasi-threads", "wasmtime-wast", diff --git a/src/commands/run.rs b/src/commands/run.rs index e8e8fd94f842..740dd566fbfb 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -16,7 +16,12 @@ use wasmtime_cli_flags::{CommonOptions, WasiModules}; use wasmtime_wasi::maybe_exit_on_error; use wasmtime_wasi::sync::{ambient_authority, Dir, TcpListener, WasiCtxBuilder}; -#[cfg(any(feature = "wasi-crypto", feature = "wasi-nn", feature = "wasi-threads"))] +#[cfg(any( + feature = "wasi-crypto", + feature = "wasi-nn", + feature = "wasi-threads", + feature = "wasi-http" +))] use std::sync::Arc; #[cfg(feature = "wasi-nn")] @@ -28,6 +33,9 @@ use wasmtime_wasi_crypto::WasiCryptoCtx; #[cfg(feature = "wasi-threads")] use wasmtime_wasi_threads::WasiThreadsCtx; +#[cfg(feature = "wasi-http")] +use wasmtime_wasi_http::WasiHttp; + fn parse_module(s: &OsStr) -> anyhow::Result { // Do not accept wasmtime subcommand names as the module name match s.to_str() { @@ -525,6 +533,8 @@ struct Host { wasi_nn: Option>, #[cfg(feature = "wasi-threads")] wasi_threads: Option>>, + #[cfg(feature = "wasi-http")] + wasi_http: Option, limits: StoreLimits, } @@ -625,6 +635,21 @@ fn populate_with_wasi( } } + if wasi_modules.wasi_http { + #[cfg(not(feature = "wasi-http"))] + { + bail!("Cannot enable wasi-http when the binary is not compiled with this feature."); + } + #[cfg(feature = "wasi-http")] + { + let w_http = WasiHttp::new(); + wasmtime_wasi_http::add_to_linker(linker, |host: &mut Host| { + host.wasi_http.as_mut().unwrap() + })?; + store.data_mut().wasi_http = Some(w_http); + } + } + Ok(()) }