From 2a3e22bf04b41c6116cb1e29ab7d8daeed8e63db Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 24 Jun 2024 12:30:44 -0700 Subject: [PATCH 1/3] Promote the `wasm32-wasip2` target to Tier 2 This commit promotes the `wasm32-wasip2` Rust target to tier 2 as proposed in rust-lang/compiler-team#760. There are two major changes in this PR: 1. The `dist-various-2` container, which already produces the other WASI targets, now has an extra target added for `wasm32-wasip2`. 2. A new `wasm-component-ld` binary is added to all host toolchains when LLD is enabled. This is the linker used for the `wasm32-wasip2` target. This new linker is added for all host toolchains to ensure that all host toolchains can produce the `wasm32-wasip2` target. This is similar to how `rust-lld` was originally included for all host toolchains to be able to produce WebAssembly output when the targets were first added. The new linker is developed [here][wasm-component-ld] and is pulled in via a crates.io-based dependency to the tree here. [wasm-component-ld]: https://github.com/bytecodealliance/wasm-component-ld --- Cargo.lock | 157 +++++++++++++++++- Cargo.toml | 4 + src/bootstrap/src/core/build_steps/compile.rs | 15 ++ src/bootstrap/src/core/build_steps/tool.rs | 28 ++++ .../host-x86_64/dist-various-2/Dockerfile | 1 + src/tools/tidy/src/deps.rs | 5 + src/tools/wasm-component-ld/Cargo.toml | 13 ++ src/tools/wasm-component-ld/README.md | 62 +++++++ src/tools/wasm-component-ld/src/main.rs | 9 + 9 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 src/tools/wasm-component-ld/Cargo.toml create mode 100644 src/tools/wasm-component-ld/README.md create mode 100644 src/tools/wasm-component-ld/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index afeb9faec097e..f426c0e370402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1671,6 +1671,7 @@ dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", "rustc-std-workspace-core", + "serde", ] [[package]] @@ -1880,6 +1881,12 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "ident_case" version = "1.0.1" @@ -2109,6 +2116,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" +[[package]] +name = "lexopt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" + [[package]] name = "libc" version = "0.2.155" @@ -2623,7 +2636,7 @@ dependencies = [ "indexmap", "memchr", "ruzstd 0.5.0", - "wasmparser", + "wasmparser 0.118.2", ] [[package]] @@ -3424,7 +3437,7 @@ dependencies = [ "object 0.34.0", "regex", "similar", - "wasmparser", + "wasmparser 0.118.2", ] [[package]] @@ -3805,7 +3818,7 @@ dependencies = [ "thin-vec", "thorin-dwp", "tracing", - "wasm-encoder", + "wasm-encoder 0.200.0", "windows", ] @@ -5248,6 +5261,15 @@ dependencies = [ "color-eyre", ] +[[package]] +name = "spdx" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47317bbaf63785b53861e1ae2d11b80d6b624211d42cb20efcd210ee6f8a14bc" +dependencies = [ + "smallvec", +] + [[package]] name = "spdx-expression" version = "0.5.2" @@ -6296,6 +6318,28 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm-component-ld" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "314d932d5e84c9678751b85498b1482b2f32f185744e449d3ce0b1d400376dad" +dependencies = [ + "anyhow", + "clap", + "lexopt", + "tempfile", + "wasmparser 0.210.0", + "wat", + "wit-component", +] + +[[package]] +name = "wasm-component-ld-wrapper" +version = "0.1.0" +dependencies = [ + "wasm-component-ld", +] + [[package]] name = "wasm-encoder" version = "0.200.0" @@ -6305,6 +6349,40 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-encoder" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e3764d9d6edabd8c9e16195e177be0d20f6ab942ad18af52860f12f82bc59a" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.211.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7d931a1120ef357f32b74547646b6fa68ea25e377772b72874b131a9ed70d4" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "012729d1294907fcb0866f08460ab95426a6d0b176a599619b84cac7653452b4" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.210.0", + "wasmparser 0.210.0", +] + [[package]] name = "wasmparser" version = "0.118.2" @@ -6315,6 +6393,42 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7bbcd21e7581619d9f6ca00f8c4f08f1cacfe58bf63f83af57cd0476f1026f5" +dependencies = [ + "ahash", + "bitflags 2.5.0", + "hashbrown", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wast" +version = "211.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25506dd82d00da6b14a87436b3d52b1d264083fa79cdb72a0d1b04a8595ccaa" +dependencies = [ + "bumpalo", + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.211.1", +] + +[[package]] +name = "wat" +version = "1.211.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb716ca6c86eecac2d82541ffc39860118fc0af9309c4f2670637bea2e1bdd7d" +dependencies = [ + "wast", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6542,6 +6656,43 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-component" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a450bdb5d032acf1fa0865451fa0c6f50e62f2d31eaa8dba967c2e2d068694a4" +dependencies = [ + "anyhow", + "bitflags 2.5.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.210.0", + "wasm-metadata", + "wasmparser 0.210.0", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a965cbd439af19a4b44a54a97ab8957d86f02d01320efc9e31c1d3605c6710" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.210.0", +] + [[package]] name = "writeable" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index 93c520b0d689d..ce87a8c20b77d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "src/tools/opt-dist", "src/tools/coverage-dump", "src/tools/rustc-perf-wrapper", + "src/tools/wasm-component-ld", ] exclude = [ @@ -104,6 +105,9 @@ rustc-demangle.debug = 0 [profile.release.package.lld-wrapper] debug = 0 strip = true +[profile.release.package.wasm-component-ld-wrapper] +debug = 0 +strip = true [patch.crates-io] # See comments in `library/rustc-std-workspace-core/README.md` for what's going on diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 11a7a40453516..175ef52b33fce 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1829,6 +1829,21 @@ impl Step for Assemble { &self_contained_lld_dir.join(exe(name, target_compiler.host)), ); } + + // In addition to `rust-lld` also install `wasm-component-ld` when + // LLD is enabled. This is a relatively small binary that primarily + // delegates to the `rust-lld` binary for linking and then runs + // logic to create the final binary. This is used by the + // `wasm32-wasip2` target of Rust. + let wasm_component_ld_exe = + builder.ensure(crate::core::build_steps::tool::WasmComponentLd { + compiler: build_compiler, + target: target_compiler.host, + }); + builder.copy_link( + &wasm_component_ld_exe, + &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), + ); } if builder.config.llvm_enabled(target_compiler.host) { diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index ad92a01bce7f3..e19a05371a7f2 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -701,6 +701,34 @@ impl Step for LldWrapper { } } +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct WasmComponentLd { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for WasmComponentLd { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "wasm-component-ld", + mode: Mode::ToolStd, + path: "src/tools/wasm-component-ld", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustAnalyzer { pub compiler: Compiler, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index e3cb396b78297..962484593b4be 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -112,6 +112,7 @@ ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,wasm32-wasi ENV TARGETS=$TARGETS,wasm32-wasip1 ENV TARGETS=$TARGETS,wasm32-wasip1-threads +ENV TARGETS=$TARGETS,wasm32-wasip2 ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,x86_64-pc-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 82fa43f581fde..871855df7bebe 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -95,7 +95,12 @@ const EXCEPTIONS: ExceptionList = &[ ("self_cell", "Apache-2.0"), // rustc (fluent translations) ("snap", "BSD-3-Clause"), // rustc ("wasm-encoder", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wasm-metadata", "Apache-2.0 WITH LLVM-exception"), // rustc ("wasmparser", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wast", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wat", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wit-component", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wit-parser", "Apache-2.0 WITH LLVM-exception"), // rustc // tidy-alphabetical-end ]; diff --git a/src/tools/wasm-component-ld/Cargo.toml b/src/tools/wasm-component-ld/Cargo.toml new file mode 100644 index 0000000000000..91ff19ad9fcb0 --- /dev/null +++ b/src/tools/wasm-component-ld/Cargo.toml @@ -0,0 +1,13 @@ +# See the `README.md` in this directory for what this tool is. + +[package] +name = "wasm-component-ld-wrapper" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "wasm-component-ld" +path = "src/main.rs" + +[dependencies] +wasm-component-ld = "0.5.4" diff --git a/src/tools/wasm-component-ld/README.md b/src/tools/wasm-component-ld/README.md new file mode 100644 index 0000000000000..54608a2dea1d4 --- /dev/null +++ b/src/tools/wasm-component-ld/README.md @@ -0,0 +1,62 @@ +# `wasm-component-ld` + +This wrapper is a wrapper around the [`wasm-component-ld`] crates.io crate. That +crate. That crate is itself a thin wrapper around two pieces: + +* `wasm-ld` - the LLVM-based linker distributed as part of LLD and packaged in + Rust as `rust-lld`. +* [`wit-component`] - a Rust crate for creating a [WebAssembly Component] from a + core wasm module. + +This linker is used for Rust's `wasm32-wasip2` target to natively output a +component instead of a core WebAssembly module, unlike other WebAssembly +targets. If you're confused about any of this here's an FAQ-style explanation of +what's going on here: + +* **What's a component?** - It's a proposal to the WebAssembly standard + primarily developed at this time by out-of-browser use cases of WebAssembly. + You can find high-level documentation [here][component docs]. + +* **What's WASIp2?** - Not to be confused with WASIp1, WASIp0, + `wasi_snapshot_preview1`, or `wasi_unstable`, it's a version of WASI. Released + in January 2024 it's the first version of WASI defined in terms of the + component model. + +* **Why does this need its own linker?** - like any target that Rust has the + `wasm32-wasip2` target needs a linker. What makes this different from other + WebAssembly targets is that WASIp2 is defined at the component level, not core + WebAssembly level. This means that filesystem functions take a `string` + instead of `i32 i32`, for example. This means that the raw output of LLVM and + `wasm-ld`, a core WebAssembly module, is not suitable. + +* **Isn't writing a linker really hard?** - Generally, yes, but this linker + works by first asking `wasm-ld` to do all the hard work. It invokes `wasm-ld` + and then uses the output core WebAssembly module to create a component. + +* **How do you create a component from a core module?** - this is the purpose of + the [`wit-component`] crate, notably the `ComponentEncoder` type. This uses + component type information embedded in the core module and a general set of + conventions/guidelines with what the core module imports/exports. A component + is then hooked up to codify all of these conventions in a component itself. + +* **Why not require users to run `wit-component` themselves?** - while possible + it adds friction to the usage `wasm32-wasip2` target. More importantly though + the "module only" output of the `wasm32-wasip2` target is not ready right now. + The standard library still imports from `wasi_snapshot_preview1` and it will + take time to migrate all usage to WASIp2. + +* **What exactly does this linker do?** - the `wasm-component-ld` has the same + CLI interface and flags as `wasm-ld`, plus some more that are + component-specific. These flags are used to forward most flags to `wasm-ld` to + produce a core wasm module. After the core wasm module is produced the + `wit-component` crate will read custom sections in the final binary which + contain component type information. After merging all this type information + together a component is produced which wraps the core module. + +If you've got any other questions about this linker or its operation don't +hesitate to reach out to the maintainers of the `wasm32-wasip2` target. + +[`wasm-component-ld`]: https://crates.io/crates/wasm-component-ld +[`wit-component`]: https://crates.io/crates/wit-component +[WebAssembly Component]: https://github.com/webassembly/component-model +[component docs]: https://component-model.bytecodealliance.org/ diff --git a/src/tools/wasm-component-ld/src/main.rs b/src/tools/wasm-component-ld/src/main.rs new file mode 100644 index 0000000000000..caaac651c4c7b --- /dev/null +++ b/src/tools/wasm-component-ld/src/main.rs @@ -0,0 +1,9 @@ +// See the `README.md` in this directory for what this tool is. + +// The source for this crate lives at +// https://github.com/bytecodealliance/wasm-component-ld and the binary is +// independently used in other projects such as `wasi-sdk` so the `main` +// function is just reexported here to delegate. A Cargo dependency is used to +// facilitate version management in the Rust repository and work well with +// vendored/offline builds. +use wasm_component_ld::main; From 1afdd45881d39f6af434fba77ae07e68e1b62bce Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 9 Jul 2024 10:26:33 -0700 Subject: [PATCH 2/3] Update how wasm-component-ld is built Reuse preexisting macro and switch it to a "bootstrap tool" to try to resolve build issues. --- src/bootstrap/src/core/build_steps/tool.rs | 29 +--------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index e19a05371a7f2..d62166d8f0472 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -337,6 +337,7 @@ bootstrap_tool!( RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; CoverageDump, "src/tools/coverage-dump", "coverage-dump"; RustcPerfWrapper, "src/tools/rustc-perf-wrapper", "rustc-perf-wrapper"; + WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld"; ); #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -701,34 +702,6 @@ impl Step for LldWrapper { } } -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct WasmComponentLd { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for WasmComponentLd { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "wasm-component-ld", - mode: Mode::ToolStd, - path: "src/tools/wasm-component-ld", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustAnalyzer { pub compiler: Compiler, From 4cd6eee894ec87de26b8dc212363c23f1db70c62 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 9 Jul 2024 13:02:51 -0700 Subject: [PATCH 3/3] Unconditionally use stage0 build compiler --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 175ef52b33fce..5650fea1b8124 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1837,7 +1837,7 @@ impl Step for Assemble { // `wasm32-wasip2` target of Rust. let wasm_component_ld_exe = builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, + compiler: build_compiler.with_stage(0), target: target_compiler.host, }); builder.copy_link(