Use wasi-streams in the wasi-http implementation (bytecodealliance#7056)

* Start refactoring wasi-http

* Checkpoint

* Initial implementation of response future handling

* Lazily initialize response headers and body

* make wasmtime-wasi-http compile

* wasi-http wit: make a way to reject outgoing-request in outgoing-handler

before waiting for the future to resolve

* wasi: sync wit from wasi-http

* outgoing handler impl: report errors to userland

* test-programs: get wasi-http-components kicking over, delete modules and components-sync tests

wasi-http-components-sync will come back once we get done with other
stuff, but its superfulous for now. wasi-http-modules will not be
returning.

* Process headers

* Add HostIncomingBody::new

* Add trailers functions

* Add TODO for body task outline

* Rework incoming-response-consume to return a future-trailers value as well

* Fix the wit

* First cut at the worker loop

* wasi-http: change how we represent bodies/trailers, and annotate own/borrow/child throughout

* Update types_impl.rs for wit changes

* Split body management into its own module

* Checkpoint

* more work on incoming body and future trailers

* Fill out some more functions

* Implement future-trailers-{subscribe,get}

* Implement drop-future-trailers

* Rework fields, but make the borrow checker mad

* Fix borrow error

* wasi-http-tests: fix build

* test-runner: report errors with stdout/stderr properly

* fix two trivial wasi-http tests

the error type here changed from a types::Error to an
outbound_handler::Error

* Remove unnecessary drops

* Convert a `bail!` to a `todo!`

* Remove a TODO that documented the body worker structure

* fill in a bunch more of OutputBody

* Remove the custom FrameFut future in favor of using http_body_util

* Move the outgoing body types to body.rs

* Rework the handling of outgoing bodies

* Fix the `outgoing request get` test

* Avoid deadlocking the post tests

* future_incoming_request_get shouldn't delete the resource

* Fix the invalid_dnsname test

* implement drop-future-incoming-response

* Fix invalid_port and invalid_dnsname tests

* Fix the post test

* Passing a too large string to println! caused the large post test to fail

* Format

* Plumb through `between_bytes_timeout`

* Downgrade hyper

* Revert "Downgrade hyper"

This reverts commit fa0750e.

* Restore old https connection setup

* Sync the wasi and wasi-http http deps

* Fix tests

* Remove the module and component-sync tests, as they are currently not
  supported
* Fix the reference to the large_post test in the components test

* Fix wasi-http integration

* sync implementation of wasi-http

* Slightly more robust error checking

* Ignore the wasi-http cli test

prtest:full

* Consistent ignore attributes between sync and async tests

* Fix doc errors

* code motion: introduce intermediate `HostIncomingBodyBuilder` rather than a tuple

* explain design

* Turn FieldMap into a type synonym

* Tidy up some future state (bytecodealliance#7073)

Co-authored-by: Pat Hickey <phickey@fastly.com>

* body HostInputStream: report runtime errors with StreamRuntimeError

HostInputStream is designed wrong to need that in the first place. We
will fix it in a follow-up as soon as resources land.

---------

Co-authored-by: Pat Hickey <phickey@fastly.com>
Co-authored-by: Alex Crichton <alex@alexcrichton.com>

crates/test-programs/tests/wasi-http-components-sync.rs

@@ -46,12 +46,13 @@ impl WasiView for Ctx {
 }
 
 impl WasiHttpView for Ctx {
-    fn http_ctx(&self) -> &WasiHttpCtx {
-        &self.http
-    }
-    fn http_ctx_mut(&mut self) -> &mut WasiHttpCtx {
+    fn ctx(&mut self) -> &mut WasiHttpCtx {
         &mut self.http
     }
+
+    fn table(&mut self) -> &mut Table {
+        &mut self.table
+    }
 }
 
 fn instantiate_component(
@@ -60,7 +61,7 @@ fn instantiate_component(
 ) -> Result<(Store<Ctx>, Command), anyhow::Error> {
     let mut linker = Linker::new(&ENGINE);
     add_to_linker(&mut linker)?;
-    wasmtime_wasi_http::proxy::sync::add_to_linker(&mut linker)?;
+    wasmtime_wasi_http::proxy::add_to_linker(&mut linker)?;
 
     let mut store = Store::new(&ENGINE, ctx);
 
@@ -84,7 +85,7 @@ fn run(name: &str) -> anyhow::Result<()> {
             builder.env(var, val);
         }
         let wasi = builder.build(&mut table)?;
-        let http = WasiHttpCtx::new();
+        let http = WasiHttpCtx {};
 
         let (mut store, command) = instantiate_component(component, Ctx { table, wasi, http })?;
         command

crates/test-programs/tests/wasi-http-components.rs

@@ -47,10 +47,10 @@ impl WasiView for Ctx {
 }
 
 impl WasiHttpView for Ctx {
-    fn http_ctx(&self) -> &WasiHttpCtx {
-        &self.http
+    fn table(&mut self) -> &mut Table {
+        &mut self.table
     }
-    fn http_ctx_mut(&mut self) -> &mut WasiHttpCtx {
+    fn ctx(&mut self) -> &mut WasiHttpCtx {
         &mut self.http
     }
 }
@@ -85,16 +85,11 @@ async fn run(name: &str) -> anyhow::Result<()> {
             builder.env(var, val);
         }
         let wasi = builder.build(&mut table)?;
-        let http = WasiHttpCtx::new();
+        let http = WasiHttpCtx;
 
         let (mut store, command) =
             instantiate_component(component, Ctx { table, wasi, http }).await?;
-        command
-            .wasi_cli_run()
-            .call_run(&mut store)
-            .await?
-            .map_err(|()| anyhow::anyhow!("run returned a failure"))?;
-        Ok(())
+        command.wasi_cli_run().call_run(&mut store).await
     };
     r.map_err(move |trap: anyhow::Error| {
         let stdout = stdout.try_into_inner().expect("single ref to stdout");
@@ -109,7 +104,8 @@ async fn run(name: &str) -> anyhow::Result<()> {
             "error while testing wasi-tests {} with http-components",
             name
         ))
-    })?;
+    })?
+    .map_err(|()| anyhow::anyhow!("run returned an error"))?;
     Ok(())
 }
 

crates/test-programs/wasi-http-tests/src/bin/outbound_request_invalid_dnsname.rs

@@ -15,6 +15,6 @@ async fn run() {
     )
     .await;
 
-    let error = res.unwrap_err();
-    assert_eq!(error.to_string(), "Error::InvalidUrl(\"invalid dnsname\")");
+    let error = res.unwrap_err().to_string();
+    assert!(error.starts_with("Error::InvalidUrl(\"failed to lookup address information:"));
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_invalid_version.rs

@@ -17,7 +17,7 @@ async fn run() {
 
     let error = res.unwrap_err().to_string();
     if error.ne("Error::ProtocolError(\"invalid HTTP version parsed\")")
-        && error.ne("Error::ProtocolError(\"operation was canceled\")")
+        && !error.starts_with("Error::ProtocolError(\"operation was canceled")
     {
         panic!(
             r#"assertion failed: `(left == right)`

crates/test-programs/wasi-http-tests/src/bin/outbound_request_large_post.rs

@@ -23,7 +23,7 @@ async fn run() {
     .context("localhost:3000 /post large")
     .unwrap();
 
-    println!("localhost:3000 /post large: {res:?}");
+    println!("localhost:3000 /post large: {}", res.status);
     assert_eq!(res.status, 200);
     let method = res.header("x-wasmtime-test-method").unwrap();
     assert_eq!(std::str::from_utf8(method).unwrap(), "POST");

crates/test-programs/wasi-http-tests/src/bin/outbound_request_post.rs

@@ -22,5 +22,5 @@ async fn run() {
     assert_eq!(res.status, 200);
     let method = res.header("x-wasmtime-test-method").unwrap();
     assert_eq!(std::str::from_utf8(method).unwrap(), "POST");
-    assert_eq!(res.body, b"{\"foo\": \"bar\"}");
+    assert_eq!(res.body, b"{\"foo\": \"bar\"}", "invalid body returned");
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_unknown_method.rs

@@ -18,6 +18,6 @@ async fn run() {
     let error = res.unwrap_err();
     assert_eq!(
         error.to_string(),
-        "Error::InvalidUrl(\"unknown method OTHER\")"
+        "Error::Invalid(\"unknown method OTHER\")"
     );
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_unsupported_scheme.rs

@@ -18,6 +18,6 @@ async fn run() {
     let error = res.unwrap_err();
     assert_eq!(
         error.to_string(),
-        "Error::InvalidUrl(\"unsupported scheme WS\")"
+        "Error::Invalid(\"unsupported scheme WS\")"
     );
 }