From ac6739c557ddf4ee4f47a4809d0781d12951a293 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 27 Aug 2024 08:04:41 -0700 Subject: [PATCH 1/2] Re-enable testing of WASI on CI This commit updates CI to resume testing WASI. This updates the container and testing scripts from historical processes to more modern ones, e.g. downloading wasi-sdk instead of compiling a custom toolchain. This should make it easier to update in the future and keep it in sync with rust-lang/rust as well. This also required a few minor fixes such as: * The `S_IFIFO` and `S_IFMT` constants had incorrect values. * The `CLOCK_*` definitions cause `ctest2`'s parsing to panic to they're skipped with a new `#[cfg]`. * A new `langinfo.h` header was added to the list to include. * Some historically skipped checks were removed since they're no longer necessary. * Checks for `__errno_location` are disabled since that doesn't actually exist in headers. * Checks for `select` are disabled because the Rust definition got the `const`-ness swapped for the final `timeval` argument. (backport ) [ resolve conflicts - Trevor ] (cherry picked from commit 7c10562845682f605d361d1c7d0fbfa38180bc87) --- .github/workflows/full_ci.yml | 6 +---- build.rs | 1 + ci/docker/wasm32-wasi/Dockerfile | 40 ------------------------------ ci/docker/wasm32-wasi/clang.sh | 2 -- ci/docker/wasm32-wasip1/Dockerfile | 32 ++++++++++++++++++++++++ libc-test/build.rs | 35 ++++++++++++++++++-------- src/wasi.rs | 34 ++++++++++++++++--------- 7 files changed, 80 insertions(+), 70 deletions(-) delete mode 100644 ci/docker/wasm32-wasi/Dockerfile delete mode 100755 ci/docker/wasm32-wasi/clang.sh create mode 100644 ci/docker/wasm32-wasip1/Dockerfile diff --git a/.github/workflows/full_ci.yml b/.github/workflows/full_ci.yml index 6a4fbc7f4d332..3fb7616203b8a 100644 --- a/.github/workflows/full_ci.yml +++ b/.github/workflows/full_ci.yml @@ -118,11 +118,7 @@ jobs: powerpc64le-unknown-linux-gnu, s390x-unknown-linux-gnu, riscv64gc-unknown-linux-gnu, - # FIXME: A recent nightly causes a linker failure: - # https://github.com/rust-lang/rust/issues/76679 - # See this comment for more details: - # https://github.com/rust-lang/libc/pull/2225#issuecomment-880696737 - #wasm32-wasi, + wasm32-wasip1, sparc64-unknown-linux-gnu, wasm32-unknown-emscripten, x86_64-linux-android, diff --git a/build.rs b/build.rs index 076bbc0114f86..098ee0fd6d7bc 100644 --- a/build.rs +++ b/build.rs @@ -30,6 +30,7 @@ const ALLOWED_CFGS: &'static [&'static str] = &[ "libc_thread_local", "libc_underscore_const_names", "libc_union", + "libc_ctest", ]; // Extra values to allow for check-cfg. diff --git a/ci/docker/wasm32-wasi/Dockerfile b/ci/docker/wasm32-wasi/Dockerfile deleted file mode 100644 index 80ae09f75038b..0000000000000 --- a/ci/docker/wasm32-wasi/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM ubuntu:23.10 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - clang \ - curl \ - git \ - libc6-dev \ - make \ - xz-utils - -# Note that we're using `git reset --hard` to pin to a specific commit for -# verification for now. The sysroot is currently in somewhat of a state of flux -# and is expected to have breaking changes, so this is an attempt to mitigate -# those breaking changes on `libc`'s own CI -RUN git clone https://github.com/WebAssembly/wasi-libc && \ - cd wasi-libc && \ - git reset --hard ad5133410f66b93a2381db5b542aad5e0964db96 -RUN apt-get install -y --no-install-recommends llvm -RUN make -C wasi-libc install -j $(nproc) INSTALL_DIR=/wasi-libc - -RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz | \ - tar xJf - -ENV PATH=$PATH:/wasmtime-dev-x86_64-linux -COPY docker/wasm32-wasi/clang.sh /wasi-libc/bin/clang - -RUN apt-get install -y --no-install-recommends lld -RUN ln -s /usr/bin/wasm-ld-10 /usr/bin/wasm-ld -ENV PATH=$PATH:/usr/lib/llvm-10/bin - -# Of note here is our clang wrapper which just executes a normal clang -# executable with the right sysroot, and then we're sure to turn off the -# crt-static feature to ensure that the CRT that we're specifying with `clang` -# is used. -ENV CARGO_TARGET_WASM32_WASI_RUNNER=wasmtime \ - CARGO_TARGET_WASM32_WASI_LINKER=/wasi-libc/bin/clang \ - CC_wasm32_wasi=/wasi-libc/bin/clang \ - PATH=$PATH:/rust/bin \ - RUSTFLAGS=-Ctarget-feature=-crt-static diff --git a/ci/docker/wasm32-wasi/clang.sh b/ci/docker/wasm32-wasi/clang.sh deleted file mode 100755 index 83f5da5ad6d81..0000000000000 --- a/ci/docker/wasm32-wasi/clang.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -exec /usr/bin/clang --target=wasm32-wasi --sysroot /wasi-libc/sysroot "$@" diff --git a/ci/docker/wasm32-wasip1/Dockerfile b/ci/docker/wasm32-wasip1/Dockerfile new file mode 100644 index 0000000000000..9ffb85671e3d3 --- /dev/null +++ b/ci/docker/wasm32-wasip1/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:24.04 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + clang \ + xz-utils + +# Wasmtime is used to execute tests and wasi-sdk is used to compile tests. +# Download appropriate versions here and configure various flags below. +ENV WASMTIME 24.0.0 +ENV WASI_SDK 24 + +RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$WASMTIME/wasmtime-v$WASMTIME-x86_64-linux.tar.xz | \ + tar xJf - +ENV PATH=$PATH:/wasmtime-v$WASMTIME-x86_64-linux + +RUN curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_SDK/wasi-sdk-$WASI_SDK.0-x86_64-linux.deb +RUN dpkg -i ./wasi-sdk-*.deb + +# Note that `-D_WASI_EMULATED_PROCESS_CLOCKS` is used to enable access to +# clock-related defines even though they're emulated. Also note that the usage +# of `-Ctarget-feature=-crt-static` here forces usage of the external wasi-libc +# installed via `wasi-sdk` instead of the version that comes with the standard +# library. +ENV CARGO_TARGET_WASM32_WASIP1_RUNNER=wasmtime \ + CARGO_TARGET_WASM32_WASIP1_LINKER=/opt/wasi-sdk/bin/clang \ + CARGO_TARGET_WASM32_WASIP1_RUSTFLAGS="-lwasi-emulated-process-clocks -Ctarget-feature=-crt-static" \ + CC_wasm32_wasip1=/opt/wasi-sdk/bin/clang \ + CFLAGS_wasm32_wasip1=-D_WASI_EMULATED_PROCESS_CLOCKS \ + PATH=$PATH:/rust/bin diff --git a/libc-test/build.rs b/libc-test/build.rs index 30ab41202e945..8e0a9cf4ad94d 100644 --- a/libc-test/build.rs +++ b/libc-test/build.rs @@ -1530,6 +1530,7 @@ fn test_wasi(target: &str) { "dirent.h", "errno.h", "fcntl.h", + "langinfo.h", "limits.h", "locale.h", "malloc.h", @@ -1541,6 +1542,7 @@ fn test_wasi(target: &str) { "stdio.h", "stdlib.h", "string.h", + "sys/ioctl.h", "sys/resource.h", "sys/select.h", "sys/socket.h", @@ -1549,16 +1551,20 @@ fn test_wasi(target: &str) { "sys/types.h", "sys/uio.h", "sys/utsname.h", - "sys/ioctl.h", "time.h", "unistd.h", "wasi/api.h", - "wasi/libc.h", "wasi/libc-find-relpath.h", "wasi/libc-nocwd.h", + "wasi/libc.h", "wchar.h", } + // Currently `ctest2` doesn't support macros-in-static-expressions and will + // panic on them. That affects `CLOCK_*` defines in wasi to set this here + // to omit them. + cfg.cfg("libc_ctest", None); + cfg.type_name(move |ty, is_struct, is_union| match ty { "FILE" | "fd_set" | "DIR" => ty.to_string(), t if is_union => format!("union {}", t), @@ -1577,20 +1583,27 @@ fn test_wasi(target: &str) { } }); - // Looks like LLD doesn't merge duplicate imports, so if the Rust - // code imports from a module and the C code also imports from a - // module we end up with two imports of function pointers which - // import the same thing but have different function pointers - cfg.skip_fn_ptrcheck(|f| f.starts_with("__wasi")); + // These have a different and internal type in header files and are only + // used here to generate a pointer to them in bindings so skip these tests. + cfg.skip_static(|c| c.starts_with("_CLOCK_")); + + cfg.skip_fn(|f| match f { + // This function doesn't actually exist in libc's header files + "__errno_location" => true, + + // The `timeout` argument to this function is `*const` in Rust but + // mutable in C which causes a mismatch. Avoiding breakage by changing + // this in wasi-libc and instead accepting that this is slightly + // different. + "select" => true, + + _ => false, + }); // d_name is declared as a flexible array in WASI libc, so it // doesn't support sizeof. cfg.skip_field(|s, field| s == "dirent" && field == "d_name"); - // Currently Rust/clang disagree on function argument ABI, so skip these - // tests. For more info see WebAssembly/tool-conventions#88 - cfg.skip_roundtrip(|_| true); - cfg.generate("../src/lib.rs", "main.rs"); } diff --git a/src/wasi.rs b/src/wasi.rs index 36047cbaed87d..ea8d4af29aa3b 100644 --- a/src/wasi.rs +++ b/src/wasi.rs @@ -245,14 +245,14 @@ pub const AT_SYMLINK_FOLLOW: c_int = 0x2; pub const AT_REMOVEDIR: c_int = 0x4; pub const UTIME_OMIT: c_long = 0xfffffffe; pub const UTIME_NOW: c_long = 0xffffffff; -pub const S_IFIFO: mode_t = 49152; +pub const S_IFIFO: mode_t = 0o1_0000; pub const S_IFCHR: mode_t = 8192; pub const S_IFBLK: mode_t = 24576; pub const S_IFDIR: mode_t = 16384; pub const S_IFREG: mode_t = 32768; pub const S_IFLNK: mode_t = 40960; pub const S_IFSOCK: mode_t = 49152; -pub const S_IFMT: mode_t = 57344; +pub const S_IFMT: mode_t = 0o17_0000; pub const S_IRWXO: mode_t = 0x7; pub const S_IXOTH: mode_t = 0x1; pub const S_IWOTH: mode_t = 0x2; @@ -372,16 +372,26 @@ pub const _SC_PAGE_SIZE: ::c_int = _SC_PAGESIZE; pub const _SC_IOV_MAX: c_int = 60; pub const _SC_SYMLOOP_MAX: c_int = 173; -#[allow(unused_unsafe)] // `addr_of!(EXTERN_STATIC)` is now safe; remove `unsafe` when MSRV >= 1.82 -pub static CLOCK_MONOTONIC: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_MONOTONIC)) }; -#[allow(unused_unsafe)] -pub static CLOCK_PROCESS_CPUTIME_ID: clockid_t = - unsafe { clockid_t(ptr_addr_of!(_CLOCK_PROCESS_CPUTIME_ID)) }; -#[allow(unused_unsafe)] -pub static CLOCK_REALTIME: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_REALTIME)) }; -#[allow(unused_unsafe)] -pub static CLOCK_THREAD_CPUTIME_ID: clockid_t = - unsafe { clockid_t(ptr_addr_of!(_CLOCK_THREAD_CPUTIME_ID)) }; +cfg_if! { + if #[cfg(libc_ctest)] { + // skip these constants when this is active because `ctest` currently + // panics on parsing the constants below + } else { + // `addr_of!(EXTERN_STATIC)` is now safe; remove `unsafe` when MSRV >= 1.82 + #[allow(unused_unsafe)] + pub static CLOCK_MONOTONIC: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_MONOTONIC)) }; + #[allow(unused_unsafe)] + pub static CLOCK_PROCESS_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_PROCESS_CPUTIME_ID)) }; + #[allow(unused_unsafe)] + pub static CLOCK_REALTIME: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_REALTIME)) }; + #[allow(unused_unsafe)] + pub static CLOCK_THREAD_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_THREAD_CPUTIME_ID)) }; + } +} pub const ABDAY_1: ::nl_item = 0x20000; pub const ABDAY_2: ::nl_item = 0x20001; From 4d990a598bd074e0dd745fc47986f26aefd8fd1d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 28 Aug 2024 08:18:00 -0700 Subject: [PATCH 2/2] Add `wasm32-wasip2` to the test matrix on CI This is similar to #3869 except that it adds tests for `wasm32-wasip2` in addition to `wasm32-wasip1`. This is intended to eventually empower definitions from rust-lang/rust#129638 to move into this repository. (backport ) (cherry picked from commit 15f8c4445680c88def0695ecff088dfa35874301) --- .github/workflows/full_ci.yml | 1 + ci/docker/wasm32-wasip1/Dockerfile | 22 +++------------------- ci/docker/wasm32-wasip2/Dockerfile | 15 +++++++++++++++ ci/wasi.sh | 27 +++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 ci/docker/wasm32-wasip2/Dockerfile create mode 100644 ci/wasi.sh diff --git a/.github/workflows/full_ci.yml b/.github/workflows/full_ci.yml index 3fb7616203b8a..b36f4f9d2ba57 100644 --- a/.github/workflows/full_ci.yml +++ b/.github/workflows/full_ci.yml @@ -119,6 +119,7 @@ jobs: s390x-unknown-linux-gnu, riscv64gc-unknown-linux-gnu, wasm32-wasip1, + wasm32-wasip2, sparc64-unknown-linux-gnu, wasm32-unknown-emscripten, x86_64-linux-android, diff --git a/ci/docker/wasm32-wasip1/Dockerfile b/ci/docker/wasm32-wasip1/Dockerfile index 9ffb85671e3d3..e1318151d2e62 100644 --- a/ci/docker/wasm32-wasip1/Dockerfile +++ b/ci/docker/wasm32-wasip1/Dockerfile @@ -1,23 +1,7 @@ FROM ubuntu:24.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - clang \ - xz-utils - -# Wasmtime is used to execute tests and wasi-sdk is used to compile tests. -# Download appropriate versions here and configure various flags below. -ENV WASMTIME 24.0.0 -ENV WASI_SDK 24 - -RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$WASMTIME/wasmtime-v$WASMTIME-x86_64-linux.tar.xz | \ - tar xJf - -ENV PATH=$PATH:/wasmtime-v$WASMTIME-x86_64-linux - -RUN curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_SDK/wasi-sdk-$WASI_SDK.0-x86_64-linux.deb -RUN dpkg -i ./wasi-sdk-*.deb +COPY wasi.sh / +RUN bash /wasi.sh # Note that `-D_WASI_EMULATED_PROCESS_CLOCKS` is used to enable access to # clock-related defines even though they're emulated. Also note that the usage @@ -29,4 +13,4 @@ ENV CARGO_TARGET_WASM32_WASIP1_RUNNER=wasmtime \ CARGO_TARGET_WASM32_WASIP1_RUSTFLAGS="-lwasi-emulated-process-clocks -Ctarget-feature=-crt-static" \ CC_wasm32_wasip1=/opt/wasi-sdk/bin/clang \ CFLAGS_wasm32_wasip1=-D_WASI_EMULATED_PROCESS_CLOCKS \ - PATH=$PATH:/rust/bin + PATH=$PATH:/rust/bin:/wasmtime diff --git a/ci/docker/wasm32-wasip2/Dockerfile b/ci/docker/wasm32-wasip2/Dockerfile new file mode 100644 index 0000000000000..c433c491353f9 --- /dev/null +++ b/ci/docker/wasm32-wasip2/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:24.04 + +COPY wasi.sh / +RUN bash /wasi.sh + +# Note that most of these are copied from `wasm32-wasip1/Dockerfile` +# +# FIXME: the `-Clink-arg` to export `cabi_realloc` is a bug in the target +# itself, this should be fixed upstream. +ENV CARGO_TARGET_WASM32_WASIP2_RUNNER=wasmtime \ + CARGO_TARGET_WASM32_WASIP2_LINKER=/opt/wasi-sdk/bin/clang \ + CARGO_TARGET_WASM32_WASIP2_RUSTFLAGS="-lwasi-emulated-process-clocks -Ctarget-feature=-crt-static -Clink-arg=-Wl,--export,cabi_realloc" \ + CC_wasm32_wasip2=/opt/wasi-sdk/bin/clang \ + CFLAGS_wasm32_wasip2=-D_WASI_EMULATED_PROCESS_CLOCKS \ + PATH=$PATH:/rust/bin:/wasmtime diff --git a/ci/wasi.sh b/ci/wasi.sh new file mode 100644 index 0000000000000..a06c1f45af11a --- /dev/null +++ b/ci/wasi.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +apt-get update +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + clang \ + xz-utils + +# Wasmtime is used to execute tests and wasi-sdk is used to compile tests. +# Download appropriate versions here and configure various flags below. +# +# At the time of this writing wasmtime 24.0.0 is the latest release and +# wasi-sdk-24 is the latest release, that these numbers match is just +# coincidence. +wasmtime=24.0.0 +wasi_sdk=24 + +curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime/wasmtime-v$wasmtime-x86_64-linux.tar.xz | \ + tar xJf - +mv wasmtime-v$wasmtime-x86_64-linux wasmtime + +# The pre-built `*.deb` files for wasi-sdk install to `/opt/wasi-sdk` +curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$wasi_sdk/wasi-sdk-$wasi_sdk.0-x86_64-linux.deb +dpkg -i ./wasi-sdk-*.deb