From 7f2eca6a34da3e03f0ced71858bc76a986d5d886 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Wed, 17 May 2023 10:50:47 +0200 Subject: [PATCH] rewrite miri script in Rust --- src/tools/miri/miri | 361 +----------- src/tools/miri/miri-script/Cargo.lock | 459 +++++++++++++++ src/tools/miri/miri-script/Cargo.toml | 23 + src/tools/miri/miri-script/src/arg.rs | 104 ++++ src/tools/miri/miri-script/src/commands.rs | 629 +++++++++++++++++++++ src/tools/miri/miri-script/src/main.rs | 58 ++ 6 files changed, 1277 insertions(+), 357 deletions(-) create mode 100644 src/tools/miri/miri-script/Cargo.lock create mode 100644 src/tools/miri/miri-script/Cargo.toml create mode 100644 src/tools/miri/miri-script/src/arg.rs create mode 100644 src/tools/miri/miri-script/src/commands.rs create mode 100644 src/tools/miri/miri-script/src/main.rs diff --git a/src/tools/miri/miri b/src/tools/miri/miri index bccf6d835ff8f..7412df69bd67c 100755 --- a/src/tools/miri/miri +++ b/src/tools/miri/miri @@ -1,359 +1,6 @@ #!/bin/bash set -e -USAGE=$(cat <<"EOF" - COMMANDS - -./miri install : -Installs the miri driver and cargo-miri. are passed to `cargo -install`. Sets up the rpath such that the installed binary should work in any -working directory. Note that the binaries are placed in the `miri` toolchain -sysroot, to prevent conflicts with other toolchains. - -./miri build : -Just build miri. are passed to `cargo build`. - -./miri check : -Just check miri. are passed to `cargo check`. - -./miri test : -Build miri, set up a sysroot and then run the test suite. are passed -to the final `cargo test` invocation. - -./miri run : -Build miri, set up a sysroot and then run the driver with the given . -(Also respects MIRIFLAGS environment variable.) - -./miri fmt : -Format all sources and tests. are passed to `rustfmt`. - -./miri clippy : -Runs clippy on all sources. are passed to `cargo clippy`. - -./miri cargo : -Runs just `cargo ` with the Miri-specific environment variables. -Mainly meant to be invoked by rust-analyzer. - -./miri many-seeds : -Runs over and over again with different seeds for Miri. The MIRIFLAGS -variable is set to its original value appended with ` -Zmiri-seed=$SEED` for -many different seeds. The MIRI_SEEDS variable controls how many seeds are being -tried; MIRI_SEED_START controls the first seed to try. - -./miri bench : -Runs the benchmarks from bench-cargo-miri in hyperfine. hyperfine needs to be installed. - can explicitly list the benchmarks to run; by default, all of them are run. - -./miri toolchain : -Update and activate the rustup toolchain 'miri' to the commit given in the -`rust-version` file. -`rustup-toolchain-install-master` must be installed for this to work. Any extra -flags are passed to `rustup-toolchain-install-master`. - -./miri rustc-pull : -Pull and merge Miri changes from the rustc repo. Defaults to fetching the latest -rustc commit. The fetched commit is stored in the `rust-version` file, so the -next `./miri toolchain` will install the rustc that just got pulled. - -./miri rustc-push : -Push Miri changes back to the rustc repo. This will pull a copy of the rustc -history into the Miri repo, unless you set the RUSTC_GIT env var to an existing -clone of the rustc repo. - - ENVIRONMENT VARIABLES - -MIRI_SYSROOT: -If already set, the "sysroot setup" step is skipped. - -CARGO_EXTRA_FLAGS: -Pass extra flags to all cargo invocations. (Ignored by `./miri cargo`.) -EOF -) - -## We need to know which command to run and some global constants. -COMMAND="$1" -if [ -z "$COMMAND" ]; then - echo "$USAGE" - exit 1 -fi -shift -# macOS does not have a useful readlink/realpath so we have to use Python instead... -MIRIDIR=$(python3 -c 'import pathlib, sys; print(pathlib.Path(sys.argv[1]).resolve().parent.as_posix())' "$0") -# Used for rustc syncs. -JOSH_FILTER=":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri" -# Needed for `./miri bench`. -TOOLCHAIN=$(cd "$MIRIDIR"; rustup show active-toolchain | head -n 1 | cut -d ' ' -f 1) - -## Early commands, that don't do auto-things and don't want the environment-altering things happening below. -case "$COMMAND" in -toolchain) - cd "$MIRIDIR" - NEW_COMMIT=$(cat rust-version) - # Make sure rustup-toolchain-install-master is installed. - if ! which rustup-toolchain-install-master >/dev/null; then - echo "Please install rustup-toolchain-install-master by running 'cargo install rustup-toolchain-install-master'" - exit 1 - fi - # Check if we already are at that commit. - CUR_COMMIT=$(rustc +miri --version -v 2>/dev/null | grep "^commit-hash: " | cut -d " " -f 2) - if [[ "$CUR_COMMIT" == "$NEW_COMMIT" ]]; then - echo "miri toolchain is already at commit $CUR_COMMIT." - if [[ "$TOOLCHAIN" != "miri" ]]; then - rustup override set miri - fi - exit 0 - fi - # Install and setup new toolchain. - rustup toolchain uninstall miri - rustup-toolchain-install-master -n miri -c cargo -c rust-src -c rustc-dev -c llvm-tools -c rustfmt -c clippy "$@" -- "$NEW_COMMIT" - rustup override set miri - # Cleanup. - cargo clean - # Call 'cargo metadata' on the sources in case that changes the lockfile - # (which fails under some setups when it is done from inside vscode). - cargo metadata --format-version 1 --manifest-path "$(rustc --print sysroot)/lib/rustlib/rustc-src/rust/compiler/rustc/Cargo.toml" >/dev/null - # Done! - exit 0 - ;; -rustc-pull) - cd "$MIRIDIR" - FETCH_COMMIT="$1" - if [ -z "$FETCH_COMMIT" ]; then - FETCH_COMMIT=$(git ls-remote https://github.com/rust-lang/rust/ HEAD | cut -f 1) - fi - # Update rust-version file. As a separate commit, since making it part of - # the merge has confused the heck out of josh in the past. - echo "$FETCH_COMMIT" > rust-version - git commit rust-version -m "Preparing for merge from rustc" || (echo "FAILED to commit rust-version file, something went wrong"; exit 1) - # Fetch given rustc commit and note down which one that was - git fetch http://localhost:8000/rust-lang/rust.git@$FETCH_COMMIT$JOSH_FILTER.git || (echo "FAILED to fetch new commits, something went wrong"; exit 1) - git merge FETCH_HEAD --no-ff -m "Merge from rustc" || (echo "FAILED to merge new commits ($(git rev-parse FETCH_HEAD)), something went wrong"; exit 1) - exit 0 - ;; -rustc-push) - USER="$1" - BRANCH="$2" - if [ -z "$USER" ] || [ -z "$BRANCH" ]; then - echo "Usage: $0 rustc-push " - exit 1 - fi - if [ -n "$RUSTC_GIT" ]; then - # Use an existing fork for the branch updates. - cd "$RUSTC_GIT" - else - # Do this in the local Miri repo. - echo "This will pull a copy of the rust-lang/rust history into this Miri checkout, growing it by about 1GB." - read -r -p "To avoid that, abort now and set the RUSTC_GIT environment variable to an existing rustc checkout. Proceed? [y/N] " - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - cd "$MIRIDIR" - fi - # Prepare the branch. Pushing works much better if we use as base exactly - # the commit that we pulled from last time, so we use the `rust-version` - # file as a good approximation of that. - BASE=$(cat "$MIRIDIR/rust-version") - echo "Preparing $USER/rust (base: $BASE)..." - if git fetch "https://github.com/$USER/rust" "$BRANCH" &>/dev/null; then - echo "The branch '$BRANCH' seems to already exist in 'https://github.com/$USER/rust'. Please delete it and try again." - exit 1 - fi - git fetch https://github.com/rust-lang/rust $BASE - git push https://github.com/$USER/rust $BASE:refs/heads/$BRANCH -f - echo - # Do the actual push. - cd "$MIRIDIR" - echo "Pushing Miri changes..." - git push http://localhost:8000/$USER/rust.git$JOSH_FILTER.git HEAD:$BRANCH - # Do a round-trip check to make sure the push worked as expected. - echo - git fetch http://localhost:8000/$USER/rust.git@$JOSH_FILTER.git $BRANCH &>/dev/null - if [[ $(git rev-parse HEAD) != $(git rev-parse FETCH_HEAD) ]]; then - echo "ERROR: Josh created a non-roundtrip push! Do NOT merge this into rustc!" - exit 1 - else - echo "Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:" - echo " https://github.com/$USER/rust/pull/new/$BRANCH" - exit 0 - fi - ;; -many-seeds) - MIRI_SEED_START=${MIRI_SEED_START:-0} # default to 0 - MIRI_SEEDS=${MIRI_SEEDS:-256} # default to 256 - for SEED in $(seq $MIRI_SEED_START $(( $MIRI_SEED_START + $MIRI_SEEDS - 1 )) ); do - echo "Trying seed: $SEED" - MIRIFLAGS="$MIRIFLAGS -Zlayout-seed=$SEED -Zmiri-seed=$SEED" $@ || { echo "Failing seed: $SEED"; break; } - done - exit 0 - ;; -bench) - # The hyperfine to use - HYPERFINE=${HYPERFINE:-hyperfine -w 1 -m 5 --shell=none} - # Make sure we have an up-to-date Miri installed - "$0" install - # Run the requested benchmarks - if [ -z "${1+exists}" ]; then - BENCHES=( $(ls "$MIRIDIR/bench-cargo-miri" ) ) - else - BENCHES=("$@") - fi - for BENCH in "${BENCHES[@]}"; do - $HYPERFINE "cargo +$TOOLCHAIN miri run --manifest-path $MIRIDIR/bench-cargo-miri/$BENCH/Cargo.toml" - done - exit 0 - ;; -esac - -## Run the auto-things. -if [ -z "$MIRI_AUTO_OPS" ]; then - export MIRI_AUTO_OPS=42 - - # Run this first, so that the toolchain doesn't change after - # other code has run. - if [ -f "$MIRIDIR/.auto-everything" ] || [ -f "$MIRIDIR/.auto-toolchain" ] ; then - $0 toolchain - # Let's make sure to actually use that toolchain, too. - TOOLCHAIN=miri - fi - - if [ -f "$MIRIDIR/.auto-everything" ] || [ -f "$MIRIDIR/.auto-fmt" ] ; then - $0 fmt - fi - - if [ -f "$MIRIDIR/.auto-everything" ] || [ -f "$MIRIDIR/.auto-clippy" ] ; then - $0 clippy -- -D warnings - fi -fi - -## Prepare the environment -# Determine some toolchain properties -TARGET=$(rustc +$TOOLCHAIN --version --verbose | grep "^host:" | cut -d ' ' -f 2) -SYSROOT=$(rustc +$TOOLCHAIN --print sysroot) -LIBDIR=$SYSROOT/lib/rustlib/$TARGET/lib -if ! test -d "$LIBDIR"; then - echo "Something went wrong determining the library dir." - echo "I got $LIBDIR but that does not exist." - echo "Please report a bug at https://github.com/rust-lang/miri/issues." - exit 2 -fi - -# Prepare flags for cargo and rustc. -CARGO="cargo +$TOOLCHAIN" -# Share target dir between `miri` and `cargo-miri`. -if [ -z "$CARGO_TARGET_DIR" ]; then - export CARGO_TARGET_DIR="$MIRIDIR/target" -fi -# We configure dev builds to not be unusably slow. -if [ -z "$CARGO_PROFILE_DEV_OPT_LEVEL" ]; then - export CARGO_PROFILE_DEV_OPT_LEVEL=2 -fi -# Enable rustc-specific lints (ignored without `-Zunstable-options`). -export RUSTFLAGS="-Zunstable-options -Wrustc::internal -Wrust_2018_idioms -Wunused_lifetimes -Wsemicolon_in_expressions_from_macros $RUSTFLAGS" -# We set the rpath so that Miri finds the private rustc libraries it needs. -export RUSTFLAGS="-C link-args=-Wl,-rpath,$LIBDIR $RUSTFLAGS" - -## Helper functions - -# Build a sysroot and set MIRI_SYSROOT to use it. Arguments are passed to `cargo miri setup`. -build_sysroot() { - if ! MIRI_SYSROOT="$($CARGO run $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/cargo-miri/Cargo.toml -- miri setup --print-sysroot "$@")"; then - # Run it again so the user can see the error. - $CARGO run $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/cargo-miri/Cargo.toml -- miri setup "$@" - echo "'cargo miri setup' failed" - exit 1 - fi - export MIRI_SYSROOT -} - -# Prepare and set MIRI_SYSROOT. Respects `MIRI_TEST_TARGET` and takes into account -# locally built vs. distributed rustc. -find_sysroot() { - if [ -n "$MIRI_SYSROOT" ]; then - # Sysroot already set, use that. - return 0 - fi - # We need to build a sysroot. - if [ -n "$MIRI_TEST_TARGET" ]; then - build_sysroot --target "$MIRI_TEST_TARGET" - else - build_sysroot - fi -} - -## Main - -# Run command. -case "$COMMAND" in -install) - # Install binaries to the miri toolchain's sysroot so they do not interact with other toolchains. - $CARGO install $CARGO_EXTRA_FLAGS --path "$MIRIDIR" --force --root "$SYSROOT" "$@" - $CARGO install $CARGO_EXTRA_FLAGS --path "$MIRIDIR"/cargo-miri --force --root "$SYSROOT" "$@" - ;; -check) - # Check, and let caller control flags. - $CARGO check $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml --all-targets "$@" - $CARGO check $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/cargo-miri/Cargo.toml "$@" - ;; -build) - # Build, and let caller control flags. - $CARGO build $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml "$@" - $CARGO build $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/cargo-miri/Cargo.toml "$@" - ;; -test|bless) - # First build and get a sysroot. - $CARGO build $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml - find_sysroot - if [ "$COMMAND" = "bless" ]; then - export RUSTC_BLESS="Gesundheit" - fi - # Then test, and let caller control flags. - # Only in root project as `cargo-miri` has no tests. - $CARGO test $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml "$@" - ;; -run|run-dep) - # Scan for "--target" to overwrite the "MIRI_TEST_TARGET" env var so - # that we set the MIRI_SYSROOT up the right way. - FOUND_TARGET_OPT=0 - for ARG in "$@"; do - if [ "$LAST_ARG" = "--target" ]; then - # Found it! - export MIRI_TEST_TARGET="$ARG" - FOUND_TARGET_OPT=1 - break - fi - LAST_ARG="$ARG" - done - if [ "$FOUND_TARGET_OPT" = "0" ] && [ -n "$MIRI_TEST_TARGET" ]; then - # Make sure Miri actually uses this target. - MIRIFLAGS="$MIRIFLAGS --target $MIRI_TEST_TARGET" - fi - - CARGO="$CARGO --quiet" - # First build and get a sysroot. - $CARGO build $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml - find_sysroot - # Then run the actual command. - - if [ "$COMMAND" = "run-dep" ]; then - exec $CARGO test --test compiletest $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml -- --miri-run-dep-mode $MIRIFLAGS "$@" - else - exec $CARGO run $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml -- $MIRIFLAGS "$@" - fi - ;; -fmt) - find "$MIRIDIR" -not \( -name target -prune \) -name '*.rs' \ - | xargs rustfmt +$TOOLCHAIN --edition=2021 --config-path "$MIRIDIR/rustfmt.toml" "$@" - ;; -clippy) - $CARGO clippy $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/Cargo.toml --all-targets "$@" - $CARGO clippy $CARGO_EXTRA_FLAGS --manifest-path "$MIRIDIR"/cargo-miri/Cargo.toml "$@" - ;; -cargo) - # We carefully kept the working dir intact, so this will run cargo *on the workspace in the - # current working dir*, not on the main Miri workspace. That is exactly what RA needs. - $CARGO "$@" - ;; -*) - echo "Unknown command: $COMMAND" - exit 1 - ;; -esac +# Instead of doing just `cargo run --manifest-path .. $@`, we invoke miri-script binary directly. Invoking `cargo run` goes through +# rustup (that sets it's own environmental variables), which is undesirable. +cargo build --manifest-path "$(dirname "$0")"/miri-script/Cargo.toml +"$(dirname "$0")"/miri-script/target/debug/miri-script $@ \ No newline at end of file diff --git a/src/tools/miri/miri-script/Cargo.lock b/src/tools/miri/miri-script/Cargo.lock new file mode 100644 index 0000000000000..be6eea0ed5d98 --- /dev/null +++ b/src/tools/miri/miri-script/Cargo.lock @@ -0,0 +1,459 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "clap" +version = "4.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d21f9bf1b425d2968943631ec91202fe5e837264063503708b83013f8fc938" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914c8c79fb560f238ef6429439a30023c862f7a28e688c58f7203f12b29970bd" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "linux-raw-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" + +[[package]] +name = "miri-script" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "dunce", + "itertools", + "path_macro", + "rustc_version", + "shell-words", + "walkdir", + "which", + "xshell", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "path_macro" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e819bbd49d5939f682638fa54826bf1650abddcd65d000923de8ad63cc7d15" + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "xshell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962c039b3a7b16cf4e9a4248397c6585c07547412e7d6a6e035389a802dcfe90" +dependencies = [ + "xshell-macros", +] + +[[package]] +name = "xshell-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dbabb1cbd15a1d6d12d9ed6b35cc6777d4af87ab3ba155ea37215f20beab80c" diff --git a/src/tools/miri/miri-script/Cargo.toml b/src/tools/miri/miri-script/Cargo.toml new file mode 100644 index 0000000000000..197f6abd99039 --- /dev/null +++ b/src/tools/miri/miri-script/Cargo.toml @@ -0,0 +1,23 @@ +[package] +authors = ["Miri Team"] +description = "Helpers for miri maintenance" +license = "MIT OR Apache-2.0" +name = "miri-script" +repository = "https://github.com/rust-lang/miri" +version = "0.1.0" +default-run = "miri-script" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = {version = "4.2", features = ["derive", "env"]} +which = "4.4" +walkdir = "2.3" +itertools = "0.10" +path_macro = "1.0" +shell-words = "1.1" +anyhow = "1.0" +xshell = "0.2" +rustc_version = "0.4" +dunce = "1.0.4" diff --git a/src/tools/miri/miri-script/src/arg.rs b/src/tools/miri/miri-script/src/arg.rs new file mode 100644 index 0000000000000..24a5204e04397 --- /dev/null +++ b/src/tools/miri/miri-script/src/arg.rs @@ -0,0 +1,104 @@ +use clap::{Parser, Subcommand}; +use std::ffi::OsString; + +#[derive(Parser, Clone, Debug)] +#[command(author, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub commands: Subcommands, +} + +#[derive(Subcommand, Clone, Debug)] +pub enum Subcommands { + /// Installs the miri driver and cargo-miri. + /// Sets up the rpath such that the installed binary should work in any + /// working directory. Note that the binaries are placed in the `miri` toolchain + /// sysroot, to prevent conflicts with other toolchains. + Install { + /// Flags that are passed through to `cargo install`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Just build miri. + Build { + /// Flags that are passed through to `cargo build`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Just check miri. + Check { + /// Flags that are passed through to `cargo check`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Build miri, set up a sysroot and then run the test suite. + Test { + #[arg(long, default_value_t = false)] + bless: bool, + /// Flags that are passed through to `cargo test`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Build miri, set up a sysroot and then run the driver with the given . + /// (Also respects MIRIFLAGS environment variable.) + Run { + #[arg(long, default_value_t = false)] + dep: bool, + /// Flags that are passed through to `miri` + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Format all sources and tests. + Fmt { + /// Flags that are passed through to `rustfmt`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Runs clippy on all sources. + Clippy { + /// Flags that are passed through to `cargo clippy`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Runs just `cargo ` with the Miri-specific environment variables. + /// Mainly meant to be invoked by rust-analyzer. + Cargo { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Runs over and over again with different seeds for Miri. The MIRIFLAGS + /// variable is set to its original value appended with ` -Zmiri-seed=$SEED` for + /// many different seeds. + ManySeeds { + /// Starting seed. + #[clap(long, env("MIRI_SEED_START"), default_value_t = 0)] + seed_start: u64, + #[clap(long, env("MIRI_SEEDS"), default_value_t = 256)] + /// Amount of seeds to try. + seeds: u64, + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + command: Vec, + }, + /// Runs the benchmarks from bench-cargo-miri in hyperfine. hyperfine needs to be installed. + Bench { + /// List of benchmarks to run. By default all benchmarks are run. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + benches: Vec, + }, + /// Update and activate the rustup toolchain 'miri' to the commit given in the + /// `rust-version` file. + /// `rustup-toolchain-install-master` must be installed for this to work. Any extra + /// flags are passed to `rustup-toolchain-install-master`. + Toolchain { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + flags: Vec, + }, + /// Pull and merge Miri changes from the rustc repo. Defaults to fetching the latest + /// rustc commit. The fetched commit is stored in the `rust-version` file, so the + /// next `./miri toolchain` will install the rustc that just got pulled. + RustcPull { commit: Option }, + /// Push Miri changes back to the rustc repo. This will pull a copy of the rustc + /// history into the Miri repo, unless you set the RUSTC_GIT env var to an existing + /// clone of the rustc repo. + RustcPush { github_user: String, branch: String }, +} diff --git a/src/tools/miri/miri-script/src/commands.rs b/src/tools/miri/miri-script/src/commands.rs new file mode 100644 index 0000000000000..121b50678045b --- /dev/null +++ b/src/tools/miri/miri-script/src/commands.rs @@ -0,0 +1,629 @@ +use std::collections::BTreeMap; + +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, bail, Context, Result}; +use dunce::canonicalize; +use path_macro::path; +use xshell::{cmd, Shell}; + +use walkdir::WalkDir; + +use crate::arg::Subcommands; + +/// Used for rustc syncs. +const JOSH_FILTER: &str = + ":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri"; + +fn detect_miri_dir() -> std::io::Result { + const MIRI_SCRIPT_ROOT_DIR: &str = env!("CARGO_MANIFEST_DIR"); + Ok(canonicalize(MIRI_SCRIPT_ROOT_DIR)?.parent().unwrap().into()) +} + +/// Queries an active toolchain for `dir` via `rustup`. +fn get_active_toolchain(dir: &Path) -> Result { + let sh = Shell::new()?; + sh.change_dir(dir); + let stdout = cmd!(sh, "rustup show active-toolchain").read()?; + Ok(stdout.split_whitespace().next().context("Could not obtain active Rust toolchain")?.into()) +} + +#[derive(Clone, Debug)] +pub(super) struct MiriRunner<'a> { + /// miri_dir is the root of the miri repository checkout we are working in. + miri_dir: PathBuf, + /// active_toolchain is passed as `+toolchain` argument to cargo/rustc invocations. + active_toolchain: String, + cargo_extra_flags: Vec, + command: &'a super::Subcommands, + /// Environment variables passed to child processes. + env: BTreeMap, + /// Additional variables used by environment-altering commands. + /// These should be accessed by corresponding methods (e.g. `sysroot()`) and not directly. + sysroot: Option, +} + +fn shell_with_parent_env() -> Result { + let sh = Shell::new()?; + // xshell does not propagate parent's env variables by default. + for (k, v) in std::env::vars_os() { + sh.set_var(k, v); + } + Ok(sh) +} + +impl MiriRunner<'_> { + pub(super) fn exec(command: &super::Subcommands) -> Result<()> { + Self::exec_inner(command, true) + } + fn exec_inner(command: &super::Subcommands, run_auto_things: bool) -> Result<()> { + let miri_dir = detect_miri_dir()?; + let active_toolchain = get_active_toolchain(&miri_dir)?; + let config = command.get_config(&miri_dir); + // CARGO_EXTRA_FLAGS do not have to be a valid UTF-8, but that's what shell_words' expects. + let cargo_extra_flags = std::env::var("CARGO_EXTRA_FLAGS").unwrap_or_default(); + let cargo_extra_flags = shell_words::split(&cargo_extra_flags)?; + let env = BTreeMap::new(); + + let mut runner = MiriRunner { + miri_dir, + active_toolchain, + command, + env, + cargo_extra_flags, + sysroot: None, + }; + if let Some(config) = config { + // Run the auto-things. + if run_auto_things { + if config.toolchain { + // Run this first, so that the toolchain doesn't change after + // other code has run. + let command = Subcommands::Toolchain { flags: vec![] }; + Self::exec_inner(&command, false)?; + // Let's make sure to actually use that toolchain, too. + runner.active_toolchain = "miri".to_owned(); + } + if config.fmt { + let command = Subcommands::Fmt { flags: vec![] }; + Self::exec_inner(&command, false)?; + } + if config.clippy { + let command = Subcommands::Clippy { + flags: ["--", "-D", "warnings"].into_iter().map(OsString::from).collect(), + }; + Self::exec_inner(&command, false)?; + } + } + + // Prepare the environment + // Determine some toolchain properties + let libdir = runner.libdir()?; + if !libdir.exists() { + println!("Something went wrong determining the library dir."); + println!("I got {} but that does not exist.", libdir.display()); + println!("Please report a bug at https://github.com/rust-lang/miri/issues."); + std::process::exit(2); + } + // Share target dir between `miri` and `cargo-miri`. + let target_dir = std::env::var_os("CARGO_TARGET_DIR") + .filter(|val| !val.is_empty()) + .unwrap_or_else(|| { + let target_dir = path!(runner.miri_dir / "target"); + target_dir.into() + }); + runner.set_env("CARGO_TARGET_DIR", target_dir); + + // We configure dev builds to not be unusably slow. + let devel_opt_level = std::env::var_os("CARGO_PROFILE_DEV_OPT_LEVEL") + .filter(|val| !val.is_empty()) + .unwrap_or_else(|| "2".into()); + runner.set_env("CARGO_PROFILE_DEV_OPT_LEVEL", devel_opt_level); + let rustflags = { + let env = std::env::var_os("RUSTFLAGS"); + let mut flags_with_warnings = OsString::from( + "-Zunstable-options -Wrustc::internal -Wrust_2018_idioms -Wunused_lifetimes -Wsemicolon_in_expressions_from_macros ", + ); + if let Some(value) = env { + flags_with_warnings.push(value); + } + // We set the rpath so that Miri finds the private rustc libraries it needs. + let mut flags_with_compiler_settings = OsString::from("-C link-args=-Wl,-rpath,"); + flags_with_compiler_settings.push(&libdir); + flags_with_compiler_settings.push(flags_with_warnings); + flags_with_compiler_settings + }; + runner.set_env("RUSTFLAGS", rustflags); + } + runner.execute() + } + fn execute(&mut self) -> Result<()> { + // Run command. + match self.command { + Subcommands::Install { flags } => self.install(flags), + Subcommands::Build { flags } => self.build(flags), + Subcommands::Check { flags } => self.check(flags), + Subcommands::Test { bless, flags } => self.test(*bless, flags), + Subcommands::Run { dep, flags } => self.run(*dep, flags), + Subcommands::Fmt { flags } => self.fmt(flags), + Subcommands::Clippy { flags } => self.clippy(flags), + Subcommands::Cargo { flags } => self.cargo(flags), + Subcommands::ManySeeds { command, seed_start, seeds } => + self.many_seeds(command, *seed_start, *seeds), + Subcommands::Bench { benches } => self.bench(benches), + Subcommands::Toolchain { flags } => self.toolchain(flags), + Subcommands::RustcPull { commit } => self.rustc_pull(commit.clone()), + Subcommands::RustcPush { github_user, branch } => self.rustc_push(github_user, branch), + } + } + + fn set_env( + &mut self, + key: impl Into, + value: impl Into, + ) -> Option { + self.env.insert(key.into(), value.into()) + } + + /// Prepare and set MIRI_SYSROOT. Respects `MIRI_TEST_TARGET` and takes into account + /// locally built vs. distributed rustc. + fn find_miri_sysroot(&mut self) -> Result<()> { + let current_sysroot = std::env::var_os("MIRI_SYSROOT").unwrap_or_default(); + + if !current_sysroot.is_empty() { + // Sysroot already set, use that. + let current_value = self.set_env("MIRI_SYSROOT", ¤t_sysroot); + assert!(current_value.is_none() || current_value.unwrap() == current_sysroot); + return Ok(()); + } + // We need to build a sysroot. + let target = std::env::var_os("MIRI_TEST_TARGET").filter(|target| !target.is_empty()); + let sysroot = self.build_miri_sysroot(target.as_deref())?; + self.set_env("MIRI_SYSROOT", sysroot); + Ok(()) + } + + /// Build a sysroot and set MIRI_SYSROOT to use it. Arguments are passed to `cargo miri setup`. + fn build_miri_sysroot(&self, target: Option<&OsStr>) -> Result { + let manifest_path = path!(self.miri_dir / "cargo-miri" / "Cargo.toml"); + let Self { active_toolchain, cargo_extra_flags, .. } = &self; + let target_prefix: Option<&OsStr> = target.map(|_| "--target".as_ref()); + let sh = self.shell()?; + let output = cmd!(sh, "cargo +{active_toolchain} --quiet run {cargo_extra_flags...} --manifest-path {manifest_path} -- miri setup --print-sysroot {target_prefix...} {target...}").read(); + if output.is_err() { + // Run it again (without `--print-sysroot`) so the user can see the error. + cmd!(sh, "cargo +{active_toolchain} --quiet run {cargo_extra_flags...} --manifest-path {manifest_path} -- miri setup {target_prefix...} {target...}").run().with_context(|| "`cargo miri setup` failed")?; + } + + Ok(output?) + } + fn build_package( + // Path to Cargo.toml file of a package to build. + path: &OsStr, + toolchain: impl AsRef, + extra_flags: &[String], + args: impl IntoIterator>, + ) -> Result<()> { + let sh = Shell::new()?; + cmd!(sh, "cargo +{toolchain} build {extra_flags...} --manifest-path {path} {args...}") + .run()?; + Ok(()) + } + fn shell(&self) -> Result { + let sh = shell_with_parent_env()?; + for (k, v) in &self.env { + sh.set_var(k, v); + } + + Ok(sh) + } + + fn libdir(&self) -> Result { + let sh = shell_with_parent_env()?; + let toolchain = &self.active_toolchain; + let target_output = cmd!(sh, "rustc +{toolchain} --version --verbose").read()?; + let rustc_meta = rustc_version::version_meta_for(&target_output)?; + let target = rustc_meta.host; + + let sysroot = cmd!(sh, "rustc +{toolchain} --print sysroot").read()?; + + let sysroot = PathBuf::from(sysroot); + let libdir = path!(sysroot / "lib" / "rustlib" / target / "lib"); + Ok(libdir) + } + fn sysroot(&mut self) -> Result { + if let Some(sysroot) = self.sysroot.as_ref() { + Ok(sysroot.clone()) + } else { + let sh = shell_with_parent_env()?; + let toolchain = &self.active_toolchain; + + let sysroot: PathBuf = cmd!(sh, "rustc +{toolchain} --print sysroot").read()?.into(); + self.sysroot = Some(sysroot.clone()); + Ok(sysroot) + } + } + fn install_to_dir( + &mut self, + sh: &Shell, + path: PathBuf, + args: impl IntoIterator>, + ) -> Result<()> { + let sysroot = self.sysroot()?; + let toolchain = &self.active_toolchain; + let extra_flags = &self.cargo_extra_flags; + // "--locked" to respect the Cargo.lock file if it exists. + // Install binaries to the miri toolchain's sysroot so they do not interact with other toolchains. + cmd!(sh, "cargo +{toolchain} install {extra_flags...} --path {path} --force --root {sysroot} {args...}").run()?; + Ok(()) + } +} + +impl MiriRunner<'_> { + fn bench(&self, benches: &[OsString]) -> Result<()> { + // The hyperfine to use + let hyperfine = std::env::var("HYPERFINE"); + let hyperfine = hyperfine.as_deref().unwrap_or("hyperfine -w 1 -m 5 --shell=none"); + let hyperfine = shell_words::split(hyperfine).unwrap(); + let Some((program_name, args)) = hyperfine.split_first() else { + bail!("Expected HYPERFINE environment variable to be non-empty"); + }; + // Make sure we have an up-to-date Miri installed + Self::exec_inner(&Subcommands::Install { flags: vec![] }, false)?; + let benches_dir = path!(self.miri_dir / "bench-cargo-miri"); + let benches = if benches.is_empty() { + std::fs::read_dir(&benches_dir)? + .filter_map(|path| { + path.ok() + .filter(|dir| dir.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .map(|p| p.file_name()) + }) + .collect() + } else { + benches.to_owned() + }; + let sh = shell_with_parent_env()?; + let toolchain = &self.active_toolchain; + // Run the requested benchmarks + for bench in benches { + let current_bench_dir = path!(benches_dir / bench / "Cargo.toml"); + cmd!( + sh, + "{program_name} {args...} 'cargo +'{toolchain}' miri run --manifest-path \"'{current_bench_dir}'\"'" + ) + .run()?; + } + Ok(()) + } + + fn toolchain(&self, flags: &[OsString]) -> Result<()> { + // Make sure rustup-toolchain-install-master is installed. + which::which("rustup-toolchain-install-master").context("Please install rustup-toolchain-install-master by running 'cargo install rustup-toolchain-install-master'")?; + let sh = shell_with_parent_env()?; + sh.change_dir(&self.miri_dir); + let new_commit = Some(sh.read_file("rust-version")?.trim().to_owned()); + let current_commit = { + let rustc_info = cmd!(sh, "rustc +miri --version -v").read(); + if rustc_info.is_err() { + None + } else { + let metadata = rustc_version::version_meta_for(&rustc_info.unwrap())?; + Some( + metadata + .commit_hash + .ok_or_else(|| anyhow!("rustc metadata did not contain commit hash"))?, + ) + } + }; + // Check if we already are at that commit. + if current_commit == new_commit { + println!("miri toolchain is already at commit {}.", current_commit.unwrap()); + cmd!(sh, "rustup override set miri").run()?; + return Ok(()); + } + // Install and setup new toolchain. + cmd!(sh, "rustup toolchain uninstall miri").run()?; + + cmd!(sh, "rustup-toolchain-install-master -n miri -c cargo -c rust-src -c rustc-dev -c llvm-tools -c rustfmt -c clippy {flags...} -- {new_commit...}").run()?; + cmd!(sh, "rustup override set miri").run()?; + // Cleanup. + cmd!(sh, "cargo clean").run()?; + // Call `cargo metadata` on the sources in case that changes the lockfile + // (which fails under some setups when it is done from inside vscode). + let sysroot = cmd!(sh, "rustc --print sysroot").read()?; + let sysroot = sysroot.trim(); + cmd!(sh, "cargo metadata --format-version 1 --manifest-path {sysroot}/lib/rustlib/rustc-src/rust/compiler/rustc/Cargo.toml").ignore_stdout().run()?; + Ok(()) + } + + fn rustc_pull(&self, commit: Option) -> Result<()> { + let sh = shell_with_parent_env()?; + sh.change_dir(&self.miri_dir); + let commit: String = commit.map(Result::Ok).unwrap_or_else(|| { + let rust_repo_head = + cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?; + rust_repo_head + .split_whitespace() + .next() + .map(|front| front.trim().to_owned()) + .ok_or_else(|| anyhow!("Could not obtain Rust repo HEAD from remote.")) + })?; + // Update rust-version file. As a separate commit, since making it part of + // the merge has confused the heck out of josh in the past. + sh.write_file(path!(self.miri_dir / "rust-version"), &commit)?; + const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc"; + cmd!(sh, "git commit rust-version -m {PREPARING_COMMIT_MESSAGE}") + .run() + .context("FAILED to commit rust-version file, something went wrong")?; + // Fetch given rustc commit and note down which one that was + cmd!(sh, "git fetch http://localhost:8000/rust-lang/rust.git@{commit}{JOSH_FILTER}.git") + .run() + .context("FAILED to fetch new commits, something went wrong")?; + const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc"; + cmd!(sh, "git merge FETCH_HEAD --no-ff -m {MERGE_COMMIT_MESSAGE}") + .run() + .context("FAILED to merge new commits, something went wrong")?; + Ok(()) + } + + fn rustc_push(&self, github_user: &str, branch: &str) -> Result<()> { + let rustc_git = std::env::var_os("RUSTC_GIT"); + let working_directory = if let Some(rustc_git) = rustc_git { + rustc_git + } else { + // If rustc_git is `Some`, we'll use an existing fork for the branch updates. + // Otherwise, do this in the local Miri repo. + println!( + "This will pull a copy of the rust-lang/rust history into this Miri checkout, growing it by about 1GB." + ); + println!( + "To avoid that, abort now and set the RUSTC_GIT environment variable to an existing rustc checkout. Proceed? [y/N] " + ); + let mut answer = String::new(); + std::io::stdin().read_line(&mut answer)?; + if answer.trim().to_lowercase() != "y" { + std::process::exit(1); + } + self.miri_dir.clone().into() + }; + // Prepare the branch. Pushing works much better if we use as base exactly + // the commit that we pulled from last time, so we use the `rust-version` + // file as a good approximation of that. + let rust_version_path = path!(self.miri_dir / "rust-version"); + let base = std::fs::read_to_string(rust_version_path)?.trim().to_owned(); + println!("Preparing {github_user}/rust (base: {base})...)"); + let sh = shell_with_parent_env()?; + sh.change_dir(working_directory); + + if cmd!(sh, "git fetch https://github.com/{github_user}").read().is_ok() { + println!( + "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}'. Please delete it and try again." + ); + std::process::exit(1); + } + + cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?; + + cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}") + .run()?; + println!(); + // Do the actual push. + sh.change_dir(&self.miri_dir); + println!("Pushing miri changes..."); + cmd!( + sh, + "git push http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}" + ) + .run()?; + // Do a round-trip check to make sure the push worked as expected. + println!(); + cmd!( + sh, + "git fetch http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git {branch}" + ) + .read()?; + let head = cmd!(sh, "git rev-parse HEAD").read()?; + let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?; + if head != fetch_head { + println!("ERROR: Josh created a non-roundtrip push! Do NOT merge this into rustc!"); + std::process::exit(1); + } + println!( + "Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:" + ); + println!(" https://github.com/{github_user}/rust/pull/new/{branch}"); + Ok(()) + } + + fn install(&mut self, flags: &[OsString]) -> Result<()> { + let sh = self.shell()?; + self.install_to_dir(&sh, self.miri_dir.clone(), flags)?; + let cargo_miri_dir = path!(self.miri_dir / "cargo-miri"); + self.install_to_dir(&sh, cargo_miri_dir, flags)?; + Ok(()) + } + + fn build(&self, flags: &[OsString]) -> Result<()> { + // Build, and let caller control flags. + let miri_manifest = path!(self.miri_dir / "Cargo.toml"); + let cargo_miri_manifest = path!(self.miri_dir / "cargo-miri" / "Cargo.toml"); + Self::build_package( + miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + flags, + )?; + Self::build_package( + cargo_miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + flags, + )?; + Ok(()) + } + + fn check(&self, flags: &[OsString]) -> Result<()> { + fn check_package( + // Path to Cargo.toml file of a package to check. + path: &OsStr, + toolchain: impl AsRef, + extra_flags: &[String], + all_targets: bool, + args: impl IntoIterator>, + ) -> Result<()> { + let all_targets: Option<&OsStr> = all_targets.then_some("--all-targets".as_ref()); + let sh = Shell::new()?; + cmd!(sh, "cargo +{toolchain} check {extra_flags...} --manifest-path {path} {all_targets...} {args...}").run()?; + Ok(()) + } + // Check, and let caller control flags. + let miri_manifest = path!(self.miri_dir / "Cargo.toml"); + let cargo_miri_manifest = path!(self.miri_dir / "cargo-miri" / "Cargo.toml"); + check_package( + miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + true, + flags, + )?; + check_package( + cargo_miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + false, + flags, + )?; + Ok(()) + } + + fn test(&mut self, bless: bool, flags: &[OsString]) -> Result<()> { + let miri_manifest = path!(self.miri_dir / "Cargo.toml"); + // First build and get a sysroot. + Self::build_package( + miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + std::iter::empty::(), + )?; + self.find_miri_sysroot()?; + let extra_flags = &self.cargo_extra_flags; + // Then test, and let caller control flags. + // Only in root project as `cargo-miri` has no tests. + let sh = self.shell()?; + if bless { + sh.set_var("MIRI_BLESS", "Gesundheit"); + } + let toolchain: &OsStr = self.active_toolchain.as_ref(); + cmd!( + sh, + "cargo +{toolchain} test {extra_flags...} --manifest-path {miri_manifest} -- {flags...}" + ) + .run()?; + Ok(()) + } + + fn run(&mut self, dep: bool, flags: &[OsString]) -> Result<()> { + use itertools::Itertools; + // Scan for "--target" to overwrite the "MIRI_TEST_TARGET" env var so + // that we set the MIRI_SYSROOT up the right way. + let target = flags.iter().tuple_windows().find(|(first, _)| first == &"--target"); + if let Some((_, target)) = target { + // Found it! + self.set_env("MIRI_TEST_TARGET", target); + } else if let Some(var) = + std::env::var_os("MIRI_TEST_TARGET").filter(|target| !target.is_empty()) + { + // Make sure miri actually uses this target. + let entry = self.env.entry("MIRIFLAGS".into()).or_default(); + entry.push(" --target "); + entry.push(var); + } + // First build and get a sysroot. + let miri_manifest = path!(self.miri_dir / "Cargo.toml"); + Self::build_package( + miri_manifest.as_ref(), + &self.active_toolchain, + &self.cargo_extra_flags, + std::iter::empty::(), + )?; + self.find_miri_sysroot()?; + // Then run the actual command. + let miri_flags = self.env.get(&OsString::from("MIRIFLAGS")).cloned().unwrap_or_default(); + let miri_flags: &OsStr = miri_flags.as_ref(); + let extra_flags = &self.cargo_extra_flags; + let sh = self.shell()?; + let toolchain: &OsStr = self.active_toolchain.as_ref(); + if dep { + cmd!( + sh, + "cargo +{toolchain} --quiet test --test compiletest {extra_flags...} --manifest-path {miri_manifest} -- --miri-run-dep-mode {miri_flags} {flags...}" + ).run()?; + } else { + cmd!( + sh, + "cargo +{toolchain} --quiet run {extra_flags...} --manifest-path {miri_manifest} -- {miri_flags} {flags...}" + ).run()?; + } + Ok(()) + } + + fn fmt(&self, flags: &[OsString]) -> Result<()> { + let toolchain = &self.active_toolchain; + let config_path = path!(self.miri_dir / "rustfmt.toml"); + let sh = self.shell()?; + for item in WalkDir::new(&self.miri_dir).into_iter().filter_entry(|entry| { + let name: String = entry.file_name().to_string_lossy().into(); + let ty = entry.file_type(); + if ty.is_file() { + name.ends_with(".rs") + } else { + // dir or symlink + &name != "target" + } + }) { + let item = item.unwrap(); // Should never panic as we've already filtered out failed entries. + if item.file_type().is_file() { + let path = item.path(); + cmd!(sh, "rustfmt +{toolchain} --edition=2021 --config-path {config_path} {flags...} {path}").quiet().run()?; + } + } + Ok(()) + } + + fn clippy(&self, flags: &[OsString]) -> Result<()> { + let toolchain_modifier = &self.active_toolchain; + let extra_flags = &self.cargo_extra_flags; + let miri_manifest = path!(self.miri_dir / "Cargo.toml"); + let sh = self.shell()?; + cmd!(sh, "cargo +{toolchain_modifier} clippy {extra_flags...} --manifest-path {miri_manifest} --all-targets -- {flags...}").run()?; + Ok(()) + } + + fn cargo(&self, flags: &[OsString]) -> Result<()> { + // We carefully kept the working dir intact, so this will run cargo *on the workspace in the + // current working dir*, not on the main Miri workspace. That is exactly what RA needs. + let toolchain_modifier = &self.active_toolchain; + let sh = self.shell()?; + cmd!(sh, "cargo +{toolchain_modifier} {flags...}").run()?; + Ok(()) + } + fn many_seeds(&self, command: &[OsString], seed_start: u64, seed_count: u64) -> Result<()> { + let seed_end = seed_start + seed_count; + assert!(!command.is_empty()); + let (command_name, trailing_args) = command.split_first().unwrap(); + let sh = shell_with_parent_env()?; + for seed in seed_start..seed_end { + println!("Trying seed: {seed}"); + let mut miriflags = std::env::var_os("MIRIFLAGS").unwrap_or_default(); + miriflags.push(format!(" -Zlayout-seed={seed} -Zmiri-seed={seed}")); + let status = + cmd!(sh, "{command_name} {trailing_args...}").env("MIRIFLAGS", miriflags).run(); + if status.is_err() { + println!("Failing seed: {seed}"); + break; + } + } + Ok(()) + } +} diff --git a/src/tools/miri/miri-script/src/main.rs b/src/tools/miri/miri-script/src/main.rs new file mode 100644 index 0000000000000..dbee923d48f8a --- /dev/null +++ b/src/tools/miri/miri-script/src/main.rs @@ -0,0 +1,58 @@ +pub(crate) mod arg; +mod commands; + +use std::path::Path; + +use anyhow::Result; +use clap::Parser; +use path_macro::path; + +use arg::Subcommands; + +struct AutoConfig { + toolchain: bool, + fmt: bool, + clippy: bool, +} + +impl Subcommands { + fn run_auto_things(&self) -> bool { + use Subcommands::*; + match self { + // Early commands, that don't do auto-things and don't want the environment-altering things happening below. + Toolchain { .. } + | RustcPull { .. } + | RustcPush { .. } + | ManySeeds { .. } + | Bench { .. } => false, + Install { .. } + | Check { .. } + | Build { .. } + | Test { .. } + | Run { .. } + | Fmt { .. } + | Clippy { .. } + | Cargo { .. } => true, + } + } + fn get_config(&self, miri_dir: &Path) -> Option { + let skip_auto_ops = std::env::var_os("MIRI_AUTO_OPS").is_some(); + if !self.run_auto_things() { + return None; + } + if skip_auto_ops { + return Some(AutoConfig { toolchain: false, fmt: false, clippy: false }); + } + + let auto_everything = path!(miri_dir / ".auto_everything").exists(); + let toolchain = auto_everything || path!(miri_dir / ".auto-toolchain").exists(); + let fmt = auto_everything || path!(miri_dir / ".auto-fmt").exists(); + let clippy = auto_everything || path!(miri_dir / ".auto-clippy").exists(); + Some(AutoConfig { toolchain, fmt, clippy }) + } +} +fn main() -> Result<()> { + let args = arg::Cli::parse(); + commands::MiriRunner::exec(&args.commands)?; + Ok(()) +}