Skip to content

Commit

Permalink
Rollup merge of #95503 - jyn514:build-single-crate, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history 
bootstrap: Allow building individual crates

This aims to be as unintrusive as possible, but did still require adding a new `tail_args` field to all `Rustc` and `Std` steps.

New library and compiler crates are added to the sysroot as they are built, since it's useful to have e.g. just alloc and not std.

Fixes #44293.
  • Loading branch information
@GuillaumeGomez
GuillaumeGomez authored Jul 5, 2022
2 parents 41ad4d9 + d0011b0 commit 0a7f2c3
Show file tree
Hide file tree
Showing 9 changed files with 226 additions and 182 deletions.
19 changes: 13 additions & 6 deletions src/bootstrap/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use std::process::{Command, Stdio};
use std::time::{Duration, Instant};

use crate::cache::{Cache, Interned, INTERNER};
use crate::compile;
use crate::config::{SplitDebuginfo, TargetSelection};
use crate::dist;
use crate::doc;
Expand All @@ -26,6 +25,7 @@ use crate::tool::{self, SourceType};
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
use crate::EXTRA_CHECK_CFGS;
use crate::{check, Config};
use crate::{compile, Crate};
use crate::{Build, CLang, DocTests, GitRepo, Mode};

pub use crate::Compiler;
Expand Down Expand Up @@ -304,9 +304,7 @@ impl StepDescription {
if paths.is_empty() || builder.config.include_default_paths {
for (desc, should_run) in v.iter().zip(&should_runs) {
if desc.default && should_run.is_really_default() {
for pathset in &should_run.paths {
desc.maybe_run(builder, vec![pathset.clone()]);
}
desc.maybe_run(builder, should_run.paths.iter().cloned().collect());
}
}
}
Expand Down Expand Up @@ -424,8 +422,16 @@ impl<'a> ShouldRun<'a> {
/// any of its (local) dependencies.
///
/// `make_run` will be called a single time with all matching command-line paths.
pub fn krate(mut self, name: &str) -> Self {
for krate in self.builder.in_tree_crates(name, None) {
pub fn crate_or_deps(self, name: &str) -> Self {
let crates = self.builder.in_tree_crates(name, None);
self.crates(crates)
}

/// Indicates it should run if the command-line selects any of the given crates.
///
/// `make_run` will be called a single time with all matching command-line paths.
pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self {
for krate in crates {
let path = krate.local_path(self.builder);
self.paths.insert(PathSet::one(path, self.kind));
}
Expand Down Expand Up @@ -581,6 +587,7 @@ impl<'a> Builder<'a> {
match kind {
Kind::Build => describe!(
compile::Std,
compile::Rustc,
compile::Assemble,
compile::CodegenBackend,
compile::StartupObjects,
Expand Down
125 changes: 63 additions & 62 deletions src/bootstrap/builder/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,24 @@ fn check_cli<const N: usize>(paths: [&str; N]) {
);
}

macro_rules! std {
($host:ident => $target:ident, stage = $stage:literal) => {
compile::Std::new(
Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage },
TargetSelection::from_user(stringify!($target)),
)
};
}

macro_rules! rustc {
($host:ident => $target:ident, stage = $stage:literal) => {
compile::Rustc::new(
Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage },
TargetSelection::from_user(stringify!($target)),
)
};
}

#[test]
fn test_valid() {
// make sure multi suite paths are accepted
Expand Down Expand Up @@ -117,6 +135,17 @@ fn test_exclude_kind() {
assert!(run_build(&[path], config).contains::<tool::CargoTest>());
}

/// Ensure that if someone passes both a single crate and `library`, all library crates get built.
#[test]
fn alias_and_path_for_library() {
let mut cache =
run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"]));
assert_eq!(
first(cache.all::<compile::Std>()),
&[std!(A => A, stage = 0), std!(A => A, stage = 1)]
);
}

mod defaults {
use super::{configure, first, run_build};
use crate::builder::*;
Expand All @@ -130,10 +159,7 @@ mod defaults {
let a = TargetSelection::from_user("A");
assert_eq!(
first(cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
]
&[std!(A => A, stage = 0), std!(A => A, stage = 1),]
);
assert!(!cache.all::<compile::Assemble>().is_empty());
// Make sure rustdoc is only built once.
Expand All @@ -143,10 +169,7 @@ mod defaults {
// - this is the compiler it's _linked_ to, not built with.
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }],
);
assert_eq!(
first(cache.all::<compile::Rustc>()),
&[compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },]
);
assert_eq!(first(cache.all::<compile::Rustc>()), &[rustc!(A => A, stage = 0)],);
}

#[test]
Expand All @@ -155,10 +178,7 @@ mod defaults {
let mut cache = run_build(&[], config);

let a = TargetSelection::from_user("A");
assert_eq!(
first(cache.all::<compile::Std>()),
&[compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },]
);
assert_eq!(first(cache.all::<compile::Std>()), &[std!(A => A, stage = 0)]);
assert!(!cache.all::<compile::Assemble>().is_empty());
assert_eq!(
first(cache.all::<tool::Rustdoc>()),
Expand All @@ -185,10 +205,10 @@ mod defaults {
assert_eq!(
first(cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: b },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
std!(A => A, stage = 0),
std!(A => A, stage = 1),
std!(A => B, stage = 0),
std!(A => B, stage = 1),
]
);
assert_eq!(
Expand All @@ -208,10 +228,7 @@ mod defaults {
);
assert_eq!(
first(cache.all::<compile::Rustc>()),
&[
compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: b },
]
&[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),]
);
}

Expand Down Expand Up @@ -334,19 +351,18 @@ mod dist {
assert_eq!(
first(cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
std!(A => A, stage = 0),
std!(A => A, stage = 1),
std!(A => A, stage = 2),
std!(A => B, stage = 1),
std!(A => B, stage = 2),
],
);
assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
}

#[test]
fn dist_only_cross_host() {
let a = TargetSelection::from_user("A");
let b = TargetSelection::from_user("B");
let mut config = configure(&["A", "B"], &["A", "B"]);
config.docs = false;
Expand All @@ -360,10 +376,7 @@ mod dist {
);
assert_eq!(
first(cache.all::<compile::Rustc>()),
&[
compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b },
]
&[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),]
);
}

Expand Down Expand Up @@ -450,11 +463,11 @@ mod dist {
assert_eq!(
first(cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
std!(A => A, stage = 0),
std!(A => A, stage = 1),
std!(A => A, stage = 2),
std!(A => B, stage = 1),
std!(A => B, stage = 2),
]
);
assert_eq!(
Expand All @@ -474,33 +487,29 @@ mod dist {
let mut builder = Builder::new(&build);
builder.run_step_descriptions(
&Builder::get_step_descriptions(Kind::Build),
&["compiler/rustc".into(), "library/std".into()],
&["compiler/rustc".into(), "library".into()],
);

let a = TargetSelection::from_user("A");
let b = TargetSelection::from_user("B");
let c = TargetSelection::from_user("C");

assert_eq!(
first(builder.cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
std!(A => A, stage = 0),
std!(A => A, stage = 1),
std!(A => A, stage = 2),
std!(A => B, stage = 1),
std!(A => B, stage = 2),
std!(A => C, stage = 2),
]
);
assert!(!builder.cache.all::<compile::Assemble>().is_empty());
assert_eq!(builder.cache.all::<compile::Assemble>().len(), 5);
assert_eq!(
first(builder.cache.all::<compile::Rustc>()),
&[
compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b },
compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: b },
rustc!(A => A, stage = 0),
rustc!(A => A, stage = 1),
rustc!(A => A, stage = 2),
rustc!(A => B, stage = 1),
rustc!(A => B, stage = 2),
]
);
}
Expand All @@ -513,15 +522,10 @@ mod dist {
builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);

let a = TargetSelection::from_user("A");
let c = TargetSelection::from_user("C");

assert_eq!(
first(builder.cache.all::<compile::Std>()),
&[
compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
]
&[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),]
);
assert_eq!(
first(builder.cache.all::<compile::Assemble>()),
Expand All @@ -533,10 +537,7 @@ mod dist {
);
assert_eq!(
first(builder.cache.all::<compile::Rustc>()),
&[
compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },
compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a },
]
&[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),]
);
}

Expand Down
Loading

1 comment on commit 0a7f2c3

@krasimirgg
Copy link
Contributor

@krasimirgg krasimirgg commented on 0a7f2c3 Jul 6, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We started seeing a strange error to find the proc-macro crate during stage1 bootstrapping which appears to have started at this commit.
The bootstrapping command we're using is python3 x.py build library/std compiler/rustc src/tools/cargo.
The errors look like this:

...
Building stage1 compiler artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu)
...
error[E0463]: can't find crate for `proc_macro`
   --> /root/.cargo/registry/src/git.luolix.top-1ecc6299db9ec823/proc-macro-hack-0.5.19/src/lib.rs:150:1
    |
150 | extern crate proc_macro;
    | ^^^^^^^^^^^^^^^^^^^^^^^^ can't find crate

error[E0463]: can't find crate for `proc_macro`
   --> /root/.cargo/registry/src/git.luolix.top-1ecc6299db9ec823/proc-macro2-1.0.37/src/lib.rs:119:1
    |
119 | extern crate proc_macro;
    | ^^^^^^^^^^^^^^^^^^^^^^^^ can't find crate

   Compiling unic-emoji-char v0.9.0
error[E0463]: can't find crate for `proc_macro`

error[E0635]: unknown feature `proc_macro_span`
  --> /root/.cargo/registry/src/git.luolix.top-1ecc6299db9ec823/proc-macro2-1.0.37/src/lib.rs:90:59
   |
90 | #![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))]
   |                                                           ^^^^^^^^^^^^^^^

This is currently affecting our experimental rust + llvm@head build bot (https://buildkite.com/llvm-project/rust-llvm-integrate-prototype/builds/11778#0181d3d0-a73a-4852-8a31-edcc608fbb6f), but this error seems more general. I've set up a reproducer on Ubuntu in Docker over at https://github.com/krasimirgg/bootstrap-proc-macro-issue, which triggers this error via python3 x.py build library/std compiler/rustc src/tools/cargo using freshly imported rust.

It seems like python3 x.py build library/std compiler/rustc triggers this error; running python3 x.py build library/std or python3 x.py build compiler/rustc by themselves works.

Please sign in to comment.