Skip to content

Commit

Permalink
fixup! ci: make scripts more uniform
Browse files Browse the repository at this point in the history
  • Loading branch information
tgross35 committed Nov 15, 2024
1 parent 897508b commit c170523
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 131 deletions.
2 changes: 1 addition & 1 deletion ci/install-musl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ rm -rf "$musl"
# Download, configure, build, and install musl-sanitized kernel headers:
kernel_header_ver="4.19.88"
curl --retry 5 -L \
"https://github.com/sabotage-linux/kernel-headers/archive/v${kernel_header_ver}.tar.gz" | \
"https://github.com/sabotage-linux/kernel-headers/archive/v${kernel_header_ver}.tar.gz" |
tar xzf -
(
cd "kernel-headers-${kernel_header_ver}"
Expand Down
67 changes: 34 additions & 33 deletions ci/install-rust.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,45 +10,46 @@ if [ -n "$TOOLCHAIN" ]; then
else
toolchain=nightly
fi

if [ "$OS" = "windows" ]; then
: "${TARGET?The TARGET environment variable must be set.}"
rustup set profile minimal
rustup update --force "$toolchain-$TARGET"
rustup default "$toolchain-$TARGET"
: "${TARGET?The TARGET environment variable must be set.}"
rustup set profile minimal
rustup update --force "$toolchain-$TARGET"
rustup default "$toolchain-$TARGET"
else
rustup set profile minimal
rustup update --force "$toolchain"
rustup default "$toolchain"
rustup set profile minimal
rustup update --force "$toolchain"
rustup default "$toolchain"
fi

if [ -n "$TARGET" ]; then
echo "Install target"
rustup target add "$TARGET"
echo "Install target"
rustup target add "$TARGET"
fi

if [ -n "$INSTALL_RUST_SRC" ]; then
echo "Install rust-src"
rustup component add rust-src
echo "Install rust-src"
rustup component add rust-src
fi

if [ "$OS" = "windows" ]; then
if [ "$ARCH_BITS" = "i686" ]; then
echo "Install MinGW32"
choco install mingw --x86 --force
fi
if [ "$ARCH_BITS" = "i686" ]; then
echo "Install MinGW32"
choco install mingw --x86 --force
fi

echo "Find GCC libraries"
gcc -print-search-dirs
/usr/bin/find "C:\ProgramData\Chocolatey" -name "crt2*"
/usr/bin/find "C:\ProgramData\Chocolatey" -name "dllcrt2*"
/usr/bin/find "C:\ProgramData\Chocolatey" -name "libmsvcrt*"
echo "Find GCC libraries"
gcc -print-search-dirs
/usr/bin/find "C:\ProgramData\Chocolatey" -name "crt2*"
/usr/bin/find "C:\ProgramData\Chocolatey" -name "dllcrt2*"
/usr/bin/find "C:\ProgramData\Chocolatey" -name "libmsvcrt*"

if [ -n "$ARCH_BITS" ]; then
echo "Fix MinGW"
for i in crt2.o dllcrt2.o libmingwex.a libmsvcrt.a ; do
cp -f "/C/ProgramData/Chocolatey/lib/mingw/tools/install/mingw$ARCH_BITS/$ARCH-w64-mingw32/lib/$i" "$(rustc --print sysroot)/lib/rustlib/$TARGET/lib"
done
fi
if [ -n "$ARCH_BITS" ]; then
echo "Fix MinGW"
for i in crt2.o dllcrt2.o libmingwex.a libmsvcrt.a ; do
cp -f "/C/ProgramData/Chocolatey/lib/mingw/tools/install/mingw$ARCH_BITS/$ARCH-w64-mingw32/lib/$i" "$(rustc --print sysroot)/lib/rustlib/$TARGET/lib"
done
fi
fi

echo "Query rust and cargo versions"
Expand All @@ -63,11 +64,11 @@ rustup show
echo "Generate lockfile"
N=5
n=0
until [ $n -ge $N ]
do
if cargo generate-lockfile; then
break
fi
n=$((n+1))
sleep 1
until [ $n -ge $N ]; do
if cargo generate-lockfile; then
break
fi

n=$((n+1))
sleep 1
done
198 changes: 110 additions & 88 deletions ci/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,104 +16,126 @@ target="$1"
# It's assume that all images, when run with two disks, will run the `run.sh`
# script from the second which we place inside.
if [ "$QEMU" != "" ]; then
tmpdir=/tmp/qemu-img-creation
mkdir -p "${tmpdir}"
tmpdir=/tmp/qemu-img-creation
mkdir -p "${tmpdir}"

if [ -z "${QEMU#*.gz}" ]; then
# image is .gz : download and uncompress it
qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" | \
gunzip -d > "${tmpdir}/${qemufile}"
if [ -z "${QEMU#*.gz}" ]; then
# image is .gz : download and uncompress it
qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" |
gunzip -d > "${tmpdir}/${qemufile}"
fi
elif [ -z "${QEMU#*.xz}" ]; then
# image is .xz : download and uncompress it
qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" |
unxz > "${tmpdir}/${qemufile}"
fi
else
# plain qcow2 image: just download it
qemufile="$(echo "${QEMU}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" >
"${tmpdir}/${qemufile}"
fi
fi
elif [ -z "${QEMU#*.xz}" ]; then
# image is .xz : download and uncompress it
qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" | \
unxz > "${tmpdir}/${qemufile}"
fi
else
# plain qcow2 image: just download it
qemufile="$(echo "${QEMU}" | sed 's/\//__/g')"
if [ ! -f "${tmpdir}/${qemufile}" ]; then
curl --retry 5 "${mirrors_url}/${QEMU}" \
> "${tmpdir}/${qemufile}"
fi
fi

# Create a mount a fresh new filesystem image that we'll later pass to QEMU.
# This will have a `run.sh` script will which use the artifacts inside to run
# on the host.
rm -f "${tmpdir}/libc-test.img"
mkdir "${tmpdir}/mount"
# Create a mount a fresh new filesystem image that we'll later pass to QEMU.
# This will have a `run.sh` script will which use the artifacts inside to run
# on the host.
rm -f "${tmpdir}/libc-test.img"
mkdir "${tmpdir}/mount"

# Do the standard rigamarole of cross-compiling an executable and then the
# script to run just executes the binary.
cargo build \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
--test main ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
rm "${CARGO_TARGET_DIR}/${target}"/debug/main-*.d
cp "${CARGO_TARGET_DIR}/${target}"/debug/main-* "${tmpdir}"/mount/libc-test
# shellcheck disable=SC2016
echo 'exec $1/libc-test' > "${tmpdir}/mount/run.sh"
# Do the standard rigamarole of cross-compiling an executable and then the
# script to run just executes the binary.
cargo build \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
--test main ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
rm "${CARGO_TARGET_DIR}/${target}"/debug/main-*.d
cp "${CARGO_TARGET_DIR}/${target}"/debug/main-* "${tmpdir}"/mount/libc-test
# shellcheck disable=SC2016
echo 'exec $1/libc-test' > "${tmpdir}/mount/run.sh"

du -sh "${tmpdir}/mount"
genext2fs \
--root "${tmpdir}/mount" \
--size-in-blocks 100000 \
"${tmpdir}/libc-test.img"
du -sh "${tmpdir}/mount"
genext2fs \
--root "${tmpdir}/mount" \
--size-in-blocks 100000 \
"${tmpdir}/libc-test.img"

# Pass -snapshot to prevent tampering with the disk images, this helps when
# running this script in development. The two drives are then passed next,
# first is the OS and second is the one we just made. Next the network is
# configured to work (I'm not entirely sure how), and then finally we turn off
# graphics and redirect the serial console output to out.log.
qemu-system-x86_64 \
-m 1024 \
-snapshot \
-drive if=virtio,file="${tmpdir}/${qemufile}" \
-drive if=virtio,file="${tmpdir}/libc-test.img" \
-net nic,model=virtio \
-net user \
-nographic \
-vga none 2>&1 | tee "${CARGO_TARGET_DIR}/out.log"
exec grep -E "^(PASSED)|(test result: ok)" "${CARGO_TARGET_DIR}/out.log"
# Pass -snapshot to prevent tampering with the disk images, this helps when
# running this script in development. The two drives are then passed next,
# first is the OS and second is the one we just made. Next the network is
# configured to work (I'm not entirely sure how), and then finally we turn off
# graphics and redirect the serial console output to out.log.
qemu-system-x86_64 \
-m 1024 \
-snapshot \
-drive if=virtio,file="${tmpdir}/${qemufile}" \
-drive if=virtio,file="${tmpdir}/libc-test.img" \
-net nic,model=virtio \
-net user \
-nographic \
-vga none 2>&1 | tee "${CARGO_TARGET_DIR}/out.log"
exec grep -E "^(PASSED)|(test result: ok)" "${CARGO_TARGET_DIR}/out.log"
fi

if [ "$target" = "s390x-unknown-linux-gnu" ]; then
# FIXME: s390x-unknown-linux-gnu often fails to test due to timeout,
# so we retry this N times.
N=5
n=0
passed=0
until [ $n -ge $N ]
do
if [ "$passed" = "0" ]; then
if cargo test --no-default-features --manifest-path libc-test/Cargo.toml --target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} ; then
passed=$((passed+1))
continue
fi
elif [ "$passed" = "1" ]; then
if cargo test --manifest-path libc-test/Cargo.toml --target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} ; then
passed=$((passed+1))
continue
fi
elif [ "$passed" = "2" ]; then
if cargo test --features extra_traits --manifest-path libc-test/Cargo.toml --target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}; then
break
fi
fi
n=$((n+1))
sleep 1
done
# FIXME: s390x-unknown-linux-gnu often fails to test due to timeout,
# so we retry this N times.
N=5
n=0
passed=0
until [ $n -ge $N ]; do
if [ "$passed" = "0" ]; then
if cargo test \
--no-default-features \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
then
passed=$((passed+1))
continue
fi
elif [ "$passed" = "1" ]; then
if cargo test \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
then
passed=$((passed+1))
continue
fi
elif [ "$passed" = "2" ]; then
if cargo test \
--features extra_traits \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
then
break
fi
fi
n=$((n+1))
sleep 1
done
else
cargo test --no-default-features --manifest-path libc-test/Cargo.toml \
--target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
cargo test \
--no-default-features \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}

cargo test --manifest-path libc-test/Cargo.toml --target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
cargo test \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}

RUST_BACKTRACE=1 cargo test --features extra_traits --manifest-path libc-test/Cargo.toml \
--target "$target" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
RUST_BACKTRACE=1 cargo test \
--features extra_traits \
--manifest-path libc-test/Cargo.toml \
--target "$target" \
${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}
fi
14 changes: 7 additions & 7 deletions ci/test-runner-linux
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

set -e

arch=$1
prog=$2
arch="$1"
prog="$2"

cd /qemu/init
echo "#!/bin/sh\n/prog --color=never" > run_prog.sh
Expand All @@ -13,11 +13,11 @@ find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz
cd ..

timeout 30s qemu-system-$arch \
-m 1024 \
-nographic \
-kernel kernel \
-initrd initrd.gz \
-append init=/run_prog.sh > output || true
-m 1024 \
-nographic \
-kernel kernel \
-initrd initrd.gz \
-append init=/run_prog.sh > output || true

# remove kernel messages
tr -d '\r' < output | grep -Ev '^\['
Expand Down
4 changes: 2 additions & 2 deletions ci/wasi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ apt-get install -y --no-install-recommends \
wasmtime=24.0.0
wasi_sdk=24

curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime/wasmtime-v$wasmtime-x86_64-linux.tar.xz | \
tar xJf -
curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime/wasmtime-v$wasmtime-x86_64-linux.tar.xz |
tar xJf -
mv wasmtime-v$wasmtime-x86_64-linux wasmtime

# The pre-built `*.deb` files for wasi-sdk install to `/opt/wasi-sdk`
Expand Down

0 comments on commit c170523

Please sign in to comment.