From c6ace5c564eae4254a39deb7ed834ad739e6e010 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 12:26:40 +0200 Subject: [PATCH 01/19] Run rustc_codegen_gcc tests in the CI --- src/bootstrap/src/core/build_steps/compile.rs | 35 ++++- src/bootstrap/src/core/build_steps/test.rs | 127 ++++++++++++++++++ src/bootstrap/src/core/builder.rs | 1 + src/ci/run.sh | 2 +- 4 files changed, 162 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 441931e415cc6..1eed534150bb6 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -45,15 +45,42 @@ pub struct Std { /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. force_recompile: bool, + extra_rust_args: &'static [&'static str], } impl Std { pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: false } + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + } } pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: true } + Self { + target, + compiler, + crates: Default::default(), + force_recompile: true, + extra_rust_args: &[], + } + } + + pub fn new_with_extra_rust_args( + compiler: Compiler, + target: TargetSelection, + extra_rust_args: &'static [&'static str], + ) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args, + } } } @@ -81,6 +108,7 @@ impl Step for Std { target: run.target, crates, force_recompile: false, + extra_rust_args: &[], }); } @@ -188,6 +216,9 @@ impl Step for Std { if target.is_synthetic() { cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); } + for rustflag in self.extra_rust_args.into_iter() { + cargo.rustflag(rustflag); + } let _guard = builder.msg( Kind::Build, diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 8c6878f61ba37..993aa6802fe1b 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -3094,3 +3094,130 @@ impl Step for CodegenCranelift { builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); } } + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenGCC { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenGCC { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + let triple = run.target.triple; + let target_supported = if triple.contains("linux") { + triple.contains("x86_64") + || triple.contains("aarch64") + || triple.contains("s390x") + || triple.contains("riscv64gc") + } else if triple.contains("darwin") || triple.contains("windows") { + triple.contains("x86_64") + } else { + false + }; + if !target_supported { + builder.info("target not supported by rustc_codegen_gcc. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + builder.info("gcc not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenGCC { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new_with_extra_rust_args( + compiler, + target, + &["-Csymbol-mangling-version=v0", "-Cpanic=abort"], + )); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + cargo.rustflag("-Cpanic=abort"); + + cargo + }; + + builder.info(&format!( + "{} GCC stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare"); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + + cargo + .arg("--") + .arg("test") + .arg("--use-system-gcc") + .arg("--use-backend") + .arg("gcc") + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) + .arg("--release") + .arg("--no-default-features") + .arg("--mini-tests") + .arg("--std-tests"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 90e09d12a9d50..44cdbe38de365 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -737,6 +737,7 @@ impl<'a> Builder<'a> { test::Debuginfo, test::UiFullDeps, test::CodegenCranelift, + test::CodegenGCC, test::Rustdoc, test::RunCoverageRustdoc, test::Pretty, diff --git a/src/ci/run.sh b/src/ci/run.sh index 9a63bb5c91c9b..0500d36e46286 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -127,7 +127,7 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" # We enable this for non-dist builders, since those aren't trying to produce # fresh binaries. We currently don't entirely support distributing a fresh From 42bdc873e5c7b5cb8bb07c31f9ba85b66f37350b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 12:30:46 +0200 Subject: [PATCH 02/19] Disable `master` feature by default when building rustc_codegen_gcc --- compiler/rustc_codegen_gcc/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/rustc_codegen_gcc/Cargo.toml b/compiler/rustc_codegen_gcc/Cargo.toml index 51fab147aa275..b0b3aeecdbdfd 100644 --- a/compiler/rustc_codegen_gcc/Cargo.toml +++ b/compiler/rustc_codegen_gcc/Cargo.toml @@ -18,7 +18,6 @@ path = "tests/lang_tests_release.rs" harness = false [features] -default = ["master"] master = ["gccjit/master"] [dependencies] From ded81de06641eb6188b58856f7d8980a99231a00 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 12:31:05 +0200 Subject: [PATCH 03/19] Fix compilation errors in rustc_codegen_gcc examples --- compiler/rustc_codegen_gcc/example/alloc_example.rs | 2 +- compiler/rustc_codegen_gcc/example/mod_bench.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_gcc/example/alloc_example.rs b/compiler/rustc_codegen_gcc/example/alloc_example.rs index f1954a30cf864..6ed8b9157f21a 100644 --- a/compiler/rustc_codegen_gcc/example/alloc_example.rs +++ b/compiler/rustc_codegen_gcc/example/alloc_example.rs @@ -18,7 +18,7 @@ extern "C" { } #[panic_handler] -fn panic_handler(_: &core::panic::PanicInfo) -> ! { +fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! { core::intrinsics::abort(); } diff --git a/compiler/rustc_codegen_gcc/example/mod_bench.rs b/compiler/rustc_codegen_gcc/example/mod_bench.rs index c60bc7fb724ea..cae911c1073d5 100644 --- a/compiler/rustc_codegen_gcc/example/mod_bench.rs +++ b/compiler/rustc_codegen_gcc/example/mod_bench.rs @@ -6,7 +6,7 @@ extern {} #[panic_handler] -fn panic_handler(_: &core::panic::PanicInfo) -> ! { +fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! { core::intrinsics::abort(); } From ead5dffeecee22fffc129bd96a92967fe68125d0 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 15:53:19 +0200 Subject: [PATCH 04/19] Fix missing error libgccjit in CI --- src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 4 ++++ src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 444e0275d48f4..1ab91b465aa6c 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ nodejs \ mingw-w64 \ + libgccjit-12-dev \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux @@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so + # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index b0eeff0c57615..00f6a12e95c65 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ xz-utils \ tidy \ + libgccjit-12-dev \ \ # Install dependencies for chromium browser gconf-service \ @@ -61,6 +62,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so + COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ From 3c58feaa085d9f082e0bff8ad72cfd9abb1ef75b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 16:29:07 +0200 Subject: [PATCH 05/19] Fix config.sh script --- compiler/rustc_codegen_gcc/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_gcc/config.sh b/compiler/rustc_codegen_gcc/config.sh index 006758e19e19c..97a7dba886a3b 100644 --- a/compiler/rustc_codegen_gcc/config.sh +++ b/compiler/rustc_codegen_gcc/config.sh @@ -25,7 +25,7 @@ else exit 1 fi -HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") +HOST_TRIPLE=$($RUSTC -vV | grep host | cut -d: -f2 | tr -d " ") # TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed. TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}" From 05a84760f65ed82acb953ca71ea4c053bed0df49 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 17:22:24 +0200 Subject: [PATCH 06/19] Fix rustc_codegen_gcc lto issue --- src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 2 ++ src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 1ab91b465aa6c..5fbf89eec031d 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -37,6 +37,8 @@ RUN sh /scripts/sccache.sh # Make `libgccjit.so` accessible. RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 00f6a12e95c65..183cc2b8d19cf 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -64,6 +64,8 @@ RUN sh /scripts/sccache.sh # Make `libgccjit.so` accessible. RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ From 7f6aa4422ca398a5b89eb94b91ac242566a9c8d3 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 18:36:18 +0200 Subject: [PATCH 07/19] Add comment explaning when to uncomment the `prepare` command code --- src/bootstrap/src/core/build_steps/test.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 993aa6802fe1b..44e5de0fa3bff 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -3063,6 +3063,7 @@ impl Step for CodegenCranelift { // FIXME handle vendoring for source tarballs before removing the --skip-test below let download_dir = builder.out.join("cg_clif_download"); + // Uncomment the `prepare` command below once vendoring is implemented. /* let mut prepare_cargo = build_cargo(); prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); @@ -3194,6 +3195,7 @@ impl Step for CodegenGCC { )); let _time = helpers::timeit(&builder); + // Uncomment the `prepare` command below once vendoring is implemented. /* let mut prepare_cargo = build_cargo(); prepare_cargo.arg("--").arg("prepare"); From edfd67b598161646dc990f29cae7b65ac9713cef Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 20:34:15 +0200 Subject: [PATCH 08/19] Pass `--sysroot` option --- compiler/rustc_codegen_gcc/config.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/rustc_codegen_gcc/config.sh b/compiler/rustc_codegen_gcc/config.sh index 97a7dba886a3b..7ae2175d41de7 100644 --- a/compiler/rustc_codegen_gcc/config.sh +++ b/compiler/rustc_codegen_gcc/config.sh @@ -54,6 +54,10 @@ if [[ -z "$BUILTIN_BACKEND" ]]; then export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS" else export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort" + + if [[ ! -z "$RUSTC_SYSROOT" ]]; then + export RUSTFLAGS="$RUSTFLAGS --sysroot $RUSTC_SYSROOT" + fi fi # FIXME(antoyo): remove once the atomic shim is gone From 4b290d40a5f152dcb71e788cb42ebf34be48e760 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 28 Oct 2023 20:35:49 +0200 Subject: [PATCH 09/19] Only run cg_gcc tests on linux x86_64 --- src/bootstrap/src/core/build_steps/test.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 44e5de0fa3bff..79a2e5366e94f 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -3121,16 +3121,8 @@ impl Step for CodegenGCC { } let triple = run.target.triple; - let target_supported = if triple.contains("linux") { - triple.contains("x86_64") - || triple.contains("aarch64") - || triple.contains("s390x") - || triple.contains("riscv64gc") - } else if triple.contains("darwin") || triple.contains("windows") { - triple.contains("x86_64") - } else { - false - }; + let target_supported = + if triple.contains("linux") { triple.contains("x86_64") } else { false }; if !target_supported { builder.info("target not supported by rustc_codegen_gcc. skipping"); return; From a1902a81d9f6e6cc0f8306ace9761962f486912b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sun, 29 Oct 2023 01:43:18 +0200 Subject: [PATCH 10/19] Force mangling version for rustc_codegen_gcc --- src/bootstrap/configure.py | 1 + src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 3 ++- src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index bfef3e672407d..626ca6cf2e7a7 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -59,6 +59,7 @@ def v(*args): o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") +o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 5fbf89eec031d..859b7c18f02fc 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -53,7 +53,8 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-15 \ --enable-llvm-link-shared \ - --set rust.thin-lto-import-instr-limit=10 + --set rust.thin-lto-import-instr-limit=10 \ + --enable-new-symbol-mangling COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 183cc2b8d19cf..82385ea15b7ca 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -87,7 +87,8 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --save-toolstates=/tmp/toolstate/toolstates.json + --save-toolstates=/tmp/toolstate/toolstates.json \ + --enable-new-symbol-mangling ENV HOST_TARGET x86_64-unknown-linux-gnu From c5ff230b559761ff0363e88599d7db6616167916 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 30 Oct 2023 13:52:38 +0100 Subject: [PATCH 11/19] Skip codegen tests in llvm-15 CI check --- src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 7d40db2ee234c..72a3cbf0596ea 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,7 +4,7 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - ../x.py --stage 1 test --skip src/tools/tidy + ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have @@ -20,7 +20,7 @@ if [[ -z "${PR_CI_JOB}" ]]; then fi # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -../x.py --stage 2 test --skip src/tools/tidy +../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have From 43290933a49f7d67be06717ac4d4bfe566f1deba Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 30 Oct 2023 18:28:16 +0100 Subject: [PATCH 12/19] Remove `libc` dependency in cg_gcc alloc_system example --- .../rustc_codegen_gcc/example/alloc_system.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_codegen_gcc/example/alloc_system.rs b/compiler/rustc_codegen_gcc/example/alloc_system.rs index 201e4c73675c9..945d34063a63c 100644 --- a/compiler/rustc_codegen_gcc/example/alloc_system.rs +++ b/compiler/rustc_codegen_gcc/example/alloc_system.rs @@ -3,7 +3,6 @@ #![no_std] #![feature(allocator_api, rustc_private)] -#![cfg_attr(any(unix, target_os = "redox"), feature(libc))] // The minimum alignment guaranteed by the architecture. This value is used to // add fast paths for low alignment values. @@ -48,7 +47,18 @@ mod realloc_fallback { } #[cfg(any(unix, target_os = "redox"))] mod platform { - extern crate libc; + mod libc { + use core::ffi::{c_void, c_int}; + + #[link(name = "c")] + extern "C" { + pub fn malloc(size: usize) -> *mut c_void; + pub fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; + pub fn calloc(nmemb: usize, size: usize) -> *mut c_void; + pub fn free(ptr: *mut u8); + pub fn posix_memalign(memptr: *mut *mut c_void, alignment: usize, size: usize) -> c_int; + } + } use core::ptr; use MIN_ALIGN; use System; @@ -82,12 +92,12 @@ mod platform { } #[inline] unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - libc::free(ptr as *mut libc::c_void) + libc::free(ptr as *mut _) } #[inline] unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { if layout.align() <= MIN_ALIGN && layout.align() <= new_size { - libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8 + libc::realloc(ptr as *mut _, new_size) as *mut u8 } else { self.realloc_fallback(ptr, layout, new_size) } From 260d91bd413c1ffe6f6e7672f1d8223c34dd21bf Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 1 Nov 2023 15:40:57 +0100 Subject: [PATCH 13/19] Add FIXME header for two comments in cg_gcc and cg_clif boostrap types --- src/bootstrap/src/core/build_steps/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 79a2e5366e94f..e2b515a30867b 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -3063,7 +3063,7 @@ impl Step for CodegenCranelift { // FIXME handle vendoring for source tarballs before removing the --skip-test below let download_dir = builder.out.join("cg_clif_download"); - // Uncomment the `prepare` command below once vendoring is implemented. + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. /* let mut prepare_cargo = build_cargo(); prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); @@ -3187,7 +3187,7 @@ impl Step for CodegenGCC { )); let _time = helpers::timeit(&builder); - // Uncomment the `prepare` command below once vendoring is implemented. + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. /* let mut prepare_cargo = build_cargo(); prepare_cargo.arg("--").arg("prepare"); From 13f7f052d855e348cb91c1336f6d7093792912a3 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 1 Nov 2023 15:48:12 +0100 Subject: [PATCH 14/19] Run codegen tests outside if not llvm-15 --- .github/workflows/ci.yml | 3 ++- .../docker/host-x86_64/x86_64-gnu-llvm-15/script.sh | 12 ++++++++++-- src/ci/docker/run.sh | 1 + src/ci/github-actions/ci.yml | 2 ++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f14cd36ce434..1bbf902cedd46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,8 +57,9 @@ jobs: os: ubuntu-20.04-4core-16gb env: {} - name: x86_64-gnu-llvm-15 + env: + SKIP_CODEGEN_TESTS: "1" os: ubuntu-20.04-16core-64gb - env: {} - name: x86_64-gnu-tools os: ubuntu-20.04-16core-64gb env: {} diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 72a3cbf0596ea..8999983e4288c 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,7 +4,11 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen + if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then + ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen + else + ../x.py --stage 1 test --skip src/tools/tidy + fi # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have @@ -20,7 +24,11 @@ if [[ -z "${PR_CI_JOB}" ]]; then fi # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen +if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then + ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen +else + ../x.py --stage 2 test --skip src/tools/tidy +fi # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 0b535532f695b..5e85bd14ea364 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -271,6 +271,7 @@ docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ + --env "SKIP_CODEGEN_TESTS=$SKIP_CODEGEN_TESTS" \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 2feb51920dff0..f5f02ba32168a 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -332,6 +332,8 @@ jobs: - name: x86_64-gnu-llvm-15 <<: *job-linux-16c + env: + SKIP_CODEGEN_TESTS: "1" - name: x86_64-gnu-tools <<: *job-linux-16c From 25a96ca0e5dc680362ebf6cb83dbab1024a527ae Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 2 Nov 2023 11:12:09 +0100 Subject: [PATCH 15/19] Don't include GCC backend if SKIP_CODEGEN_TESTS is not enabled --- src/ci/run.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ci/run.sh b/src/ci/run.sh index 0500d36e46286..2437b7f348e6b 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -126,8 +126,13 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" - # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" + if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then + # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + else + # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" + fi # We enable this for non-dist builders, since those aren't trying to produce # fresh binaries. We currently don't entirely support distributing a fresh From a141b6975b44f9cd3bc2bbc752885ffe630e4bce Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 2 Nov 2023 13:54:14 +0100 Subject: [PATCH 16/19] Rename `SKIP_CODEGEN_TESTS` into `ENABLE_GCC_CODEGEN` --- .github/workflows/ci.yml | 2 +- src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh | 4 ++-- src/ci/docker/run.sh | 2 +- src/ci/github-actions/ci.yml | 2 +- src/ci/run.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bbf902cedd46..8cc26d2995e73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: env: {} - name: x86_64-gnu-llvm-15 env: - SKIP_CODEGEN_TESTS: "1" + ENABLE_GCC_CODEGEN: "1" os: ubuntu-20.04-16core-64gb - name: x86_64-gnu-tools os: ubuntu-20.04-16core-64gb diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 8999983e4288c..b964f19fe0bac 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,7 +4,7 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen else ../x.py --stage 1 test --skip src/tools/tidy @@ -24,7 +24,7 @@ if [[ -z "${PR_CI_JOB}" ]]; then fi # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then +if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen else ../x.py --stage 2 test --skip src/tools/tidy diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 5e85bd14ea364..5021ed446581d 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -271,7 +271,7 @@ docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ - --env "SKIP_CODEGEN_TESTS=$SKIP_CODEGEN_TESTS" \ + --env "ENABLE_GCC_CODEGEN=$ENABLE_GCC_CODEGEN" \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index f5f02ba32168a..c90cd873c9e53 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -333,7 +333,7 @@ jobs: - name: x86_64-gnu-llvm-15 <<: *job-linux-16c env: - SKIP_CODEGEN_TESTS: "1" + ENABLE_GCC_CODEGEN: "1" - name: x86_64-gnu-tools <<: *job-linux-16c diff --git a/src/ci/run.sh b/src/ci/run.sh index 2437b7f348e6b..fbdd821dd3e33 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -126,7 +126,7 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" - if [[ "${SKIP_CODEGEN_TESTS}" == "1" ]]; then + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" else From 2dbe7d8d5b8408f3db2a747a4a33df80729383d2 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 2 Nov 2023 13:54:40 +0100 Subject: [PATCH 17/19] Fix invalid enabling of gcc backend in `run.sh` --- src/ci/github-actions/ci.yml | 2 +- src/ci/run.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index c90cd873c9e53..b415eb5961bb3 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -331,9 +331,9 @@ jobs: <<: *job-linux-4c - name: x86_64-gnu-llvm-15 - <<: *job-linux-16c env: ENABLE_GCC_CODEGEN: "1" + <<: *job-linux-16c - name: x86_64-gnu-tools <<: *job-linux-16c diff --git a/src/ci/run.sh b/src/ci/run.sh index fbdd821dd3e33..70956c63db59b 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -128,10 +128,10 @@ else if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" else # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" fi # We enable this for non-dist builders, since those aren't trying to produce From 30a07094a6ebdc03e23db2a33b2e24f25da55869 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 2 Nov 2023 14:14:34 +0100 Subject: [PATCH 18/19] Add comment explaining why the ENABLE_GCC_CODEGEN env variable is needed --- src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh | 4 ++++ src/ci/run.sh | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index b964f19fe0bac..2eb751ca3766d 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,6 +4,8 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen else @@ -23,6 +25,8 @@ if [[ -z "${PR_CI_JOB}" ]]; then ../x.py --stage 1 test tests/ui-fulldeps fi +# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen +# tests as it will fail them. # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen diff --git a/src/ci/run.sh b/src/ci/run.sh index 70956c63db59b..31ef55216b91c 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -126,11 +126,13 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then - # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. + # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" else - # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. + # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" fi From c890dd66b332b3e13cf413b77372ec3acd35a748 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 2 Nov 2023 21:24:28 +0100 Subject: [PATCH 19/19] Set some environment variables value only if ENABLE_GCC_CODEGEN is set --- .../host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 8 +++----- src/ci/docker/run.sh | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 859b7c18f02fc..cefdcad764397 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -35,10 +35,8 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -# Make `libgccjit.so` accessible. +# Make `libgccjit.so` accessible to the linker. RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so -# Fix rustc_codegen_gcc lto issues. -ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. @@ -53,8 +51,8 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-15 \ --enable-llvm-link-shared \ - --set rust.thin-lto-import-instr-limit=10 \ - --enable-new-symbol-mangling + $USE_NEW_MANGLING \ + --set rust.thin-lto-import-instr-limit=10 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 5021ed446581d..cedbc0390f8ff 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -267,11 +267,24 @@ fi SUMMARY_FILE=github-summary.md touch $objdir/${SUMMARY_FILE} +extra_env="" +if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then + extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1" + # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling` + # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable. + # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one + # if we run `cg_gcc` tests. + extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling" + # Fix rustc_codegen_gcc lto issues. + extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/" + echo "Setting extra environment values for docker: $extra_env" +fi + docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ - --env "ENABLE_GCC_CODEGEN=$ENABLE_GCC_CODEGEN" \ + $extra_env \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \