From 5e2de479fa07a365d89bacac473f241960970b21 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:01:34 +0300 Subject: [PATCH 01/10] Ensure compiler crate paths are generated before linking Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 628a4ece8e904..b74f6f26feaa4 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -685,18 +685,7 @@ impl Step for Rustc { target, ); - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. This is - // needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(out_dir.parent().unwrap())); - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); @@ -736,6 +725,18 @@ impl Step for Rustc { } } + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. + // + // This is needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + symlink_dir_force(&builder.config, &out, &out_dir); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + builder.run(&mut cargo.into()); if builder.paths.iter().any(|path| path.ends_with("compiler")) { From fda009c858fe2039f981be56ce4b177da1fb8703 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:02:38 +0300 Subject: [PATCH 02/10] add sanity check for compiler crate docs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index b74f6f26feaa4..99440482adc09 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -739,6 +739,14 @@ impl Step for Rustc { builder.run(&mut cargo.into()); + if !builder.config.dry_run() { + // Sanity check on linked compiler crates + for krate in &*self.crates { + let dir_name = krate.replace("-", "_"); + assert!(out.join(&*dir_name).exists()); + } + } + if builder.paths.iter().any(|path| path.ends_with("compiler")) { // For `x.py doc compiler --open`, open `rustc_middle` by default. let index = out.join("rustc_middle").join("index.html"); From d82fdb6616207dff00af563508c6e6cc5aaa4ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 28 Oct 2023 19:00:49 +0000 Subject: [PATCH 03/10] pass `CODEGEN_BACKENDS` to docker --- src/ci/docker/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index a2891ef95634d..cedbc0390f8ff 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -302,6 +302,7 @@ docker \ --env DIST_TRY_BUILD \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ + --env CODEGEN_BACKENDS \ --init \ --rm \ rust-ci \ From a9913ebed0befb09ed21725e1e30cdbd6a9c33ea Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:11:51 +0000 Subject: [PATCH 04/10] Ensure the rustc-codegen-cranelift-preview component is never empty Either generate it with the actual codegen backend dylib or omit it entirely when the cranelift backend is disabled for this build. --- src/bootstrap/src/core/build_steps/dist.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index b578c5ec29500..197ce78d8859c 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1305,6 +1305,10 @@ impl Step for CodegenBackend { return None; } + if !builder.config.rust_codegen_backends.contains(&self.backend) { + return None; + } + if self.backend == "cranelift" { if !target_supports_cranelift_backend(self.compiler.host) { builder.info("target not supported by rustc_codegen_cranelift. skipping"); @@ -1343,12 +1347,15 @@ impl Step for CodegenBackend { let backends_dst = PathBuf::from("lib").join(&backends_rel); let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; for backend in fs::read_dir(&backends_src).unwrap() { let file_name = backend.unwrap().file_name(); if file_name.to_str().unwrap().contains(&backend_name) { tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; } } + assert!(found_backend); Some(tarball.generate()) } From 451778b5517a7017eda2fea79ebb5078f11da3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 29 Oct 2023 13:12:02 +0000 Subject: [PATCH 05/10] handle dry runs in `dist::CodegenBackend` self.number_of_times_dry_runs_have_caused_issues += 1; --- src/bootstrap/src/core/build_steps/dist.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 197ce78d8859c..6e80c55c8ce3c 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1298,6 +1298,10 @@ impl Step for CodegenBackend { } fn run(self, builder: &Builder<'_>) -> Option { + if builder.config.dry_run() { + return None; + } + // This prevents rustc_codegen_cranelift from being built for "dist" // or "install" on the stable/beta channels. It is not yet stable and // should not be included. From 55b49455e79bf43ad07e3d1f1482b07ca8cf3bdf Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:43:09 +0300 Subject: [PATCH 06/10] ensure doc directory is not empty Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 99440482adc09..1ef481408f95c 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -743,7 +743,8 @@ impl Step for Rustc { // Sanity check on linked compiler crates for krate in &*self.crates { let dir_name = krate.replace("-", "_"); - assert!(out.join(&*dir_name).exists()); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); } } From 52933e0bd2006b948f174a256cca65197d2ff4a6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:49:10 +0300 Subject: [PATCH 07/10] Don't disable inline asm usage in compiler-builtins when the cranelift backend is enabled This was a leftover from when inline asm wasn't universally supported by the cranelift backend yet. It would cause the optimized inline asm intrinsics to be avoided for the standard library distributed with rustup now that we build the cranelift backend by default on nightly for several tier 1 targets. --- src/bootstrap/src/core/build_steps/compile.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 1eed534150bb6..8ab0ef61c502e 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -413,11 +413,6 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car let mut features = String::new(); - // Cranelift doesn't support `asm`. - if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { - features += " compiler-builtins-no-asm"; - } - if builder.no_std(target) == Some(true) { features += " compiler-builtins-mem"; if !target.starts_with("bpf") { From 518896441b0351b3600a38c88412a611321fe668 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 19:35:58 +0300 Subject: [PATCH 08/10] bootstrap: improve linking of tool docs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 63 ++++++++++++----------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 1ef481408f95c..8929ad1a6cd4e 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -766,10 +766,10 @@ macro_rules! tool_doc { $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? - $(in_tree = $in_tree:literal, )? - [$($extra_arg: literal),+ $(,)?] - $(,)? - ) => { + $(in_tree = $in_tree:literal ,)? + $(is_library = $is_library:expr,)? + $(crates = $crates:expr)? + ) => { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct $tool { target: TargetSelection, @@ -828,8 +828,13 @@ macro_rules! tool_doc { // Cargo uses a different directory for proc macros. builder.stage_out(compiler, Mode::ToolRustc).join("doc"), ]; + + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(&out.join(dir_name))); + })? + for out_dir in out_dirs { - t!(fs::create_dir_all(&out_dir)); symlink_dir_force(&builder.config, &out, &out_dir); } @@ -849,9 +854,13 @@ macro_rules! tool_doc { // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps"); - $( - cargo.arg($extra_arg); - )+ + if false $(|| $is_library)? { + cargo.arg("--lib"); + } + + $(for krate in $crates { + cargo.arg("-p").arg(krate); + })? cargo.rustdocflag("--document-private-items"); // Since we always pass --document-private-items, there's no need to warn about linking to private items. @@ -863,60 +872,56 @@ macro_rules! tool_doc { let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked doc directories + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + })? + } } } } } -tool_doc!( - Rustdoc, - "rustdoc-tool", - "src/tools/rustdoc", - ["-p", "rustdoc", "-p", "rustdoc-json-types"] -); +tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); tool_doc!( Rustfmt, "rustfmt-nightly", "src/tools/rustfmt", - ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"], + crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"] ); -tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]); -tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]); +tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_utils"]); +tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]); tool_doc!( Cargo, "cargo", "src/tools/cargo", rustc_tool = false, in_tree = false, - [ - "-p", + crates = [ "cargo", - "-p", "cargo-platform", - "-p", "cargo-util", - "-p", "crates-io", - "-p", "cargo-test-macro", - "-p", "cargo-test-support", - "-p", "cargo-credential", - "-p", "mdman", // FIXME: this trips a license check in tidy. - // "-p", // "resolver-tests", ] ); -tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); +tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); tool_doc!( Bootstrap, "bootstrap", "src/bootstrap", rustc_tool = false, - ["--lib", "-p", "bootstrap"] + is_library = true, + crates = ["bootstrap"] ); #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] From a348b002fbd12926fdc772b47a56563b6941ca0b Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 19:41:09 +0300 Subject: [PATCH 09/10] [tmp] act like non-try build Signed-off-by: onur-ozkan --- .github/workflows/ci.yml | 1 - src/ci/github-actions/ci.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cc26d2995e73..a1d31686c5427 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -543,7 +543,6 @@ jobs: try: name: "try - ${{ matrix.name }}" env: - DIST_TRY_BUILD: 1 CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index b415eb5961bb3..fc20d07ae51fe 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -700,7 +700,6 @@ jobs: <<: *base-ci-job name: try - ${{ matrix.name }} env: - DIST_TRY_BUILD: 1 <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust')) strategy: From 43bc2dd56f18b7fec51147e066a7c7299f8b3b4f Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 21:12:37 +0300 Subject: [PATCH 10/10] ensure existence of the tool crate dir before linking Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 30 +++++++++-------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 8929ad1a6cd4e..f2a185f70fe0c 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -685,8 +685,6 @@ impl Step for Rustc { target, ); - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); cargo.rustdocflag("--document-private-items"); @@ -713,6 +711,7 @@ impl Step for Rustc { let mut to_open = None; + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); for krate in &*self.crates { // Create all crate output directories first to make sure rustdoc uses // relative links. @@ -822,22 +821,6 @@ macro_rules! tool_doc { SourceType::Submodule }; - // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dirs = [ - builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), - // Cargo uses a different directory for proc macros. - builder.stage_out(compiler, Mode::ToolRustc).join("doc"), - ]; - - $(for krate in $crates { - let dir_name = krate.replace("-", "_"); - t!(fs::create_dir_all(&out.join(dir_name))); - })? - - for out_dir in out_dirs { - symlink_dir_force(&builder.config, &out, &out_dir); - } - // Build cargo command. let mut cargo = prepare_tool_cargo( builder, @@ -870,6 +853,17 @@ macro_rules! tool_doc { cargo.rustdocflag("--generate-link-to-definition"); cargo.rustdocflag("-Zunstable-options"); + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + })? + + // Symlink compiler docs to the output directory of rustdoc documentation. + symlink_dir_force(&builder.config, &out, &out_dir); + let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); builder.run(&mut cargo.into());