From 3a05616cb6b6df75fc54094de7f3d4b161ed8521 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 26 Sep 2019 14:44:08 -0700 Subject: [PATCH 1/3] minimize the rust-std component This splits out a rustc-dev component with the compiler crates, and keeps the status quo of default installed files on nightly. The default changing to not install compiler libraries by default is left for a future pull request. However, on stable and beta, this does remove the compiler libraries from the set of libraries installed by default, as they are never needed there (per our stability story, they "cannot" be used). --- src/bootstrap/builder.rs | 1 + src/bootstrap/dist.rs | 129 ++++++++++++++++++++------- src/tools/build-manifest/src/main.rs | 15 ++++ 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 0caf2d9b6db5a..7e3ae7f2cc907 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -443,6 +443,7 @@ impl<'a> Builder<'a> { dist::Rustc, dist::DebuggerScripts, dist::Std, + dist::RustcDev, dist::Analysis, dist::Src, dist::PlainSourceTarball, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index d9dff77a30e6b..e5a43dcb29f63 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -637,6 +637,28 @@ impl Step for DebuggerScripts { } } +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + true + } else { + false + } +} + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target).join("lib"); + t!(fs::create_dir_all(&dst)); + for (path, host) in builder.read_stamp_file(stamp) { + if !host || builder.config.build == target { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } +} + #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Std { pub compiler: Compiler, @@ -667,44 +689,19 @@ impl Step for Std { let target = self.target; let name = pkgname(builder, "rust-std"); - - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - return distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + if skip_host_target_lib(builder, compiler) { + return archive; } - // We want to package up as many target libraries as possible - // for the `rust-std` package, so if this is a host target we - // depend on librustc and otherwise we just depend on libtest. - if builder.hosts.iter().any(|t| t == target) { - builder.ensure(compile::Rustc { compiler, target }); - } else { - builder.ensure(compile::Std { compiler, target }); - } + builder.ensure(compile::Std { compiler, target }); let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); let _ = fs::remove_dir_all(&image); - let dst = image.join("lib/rustlib").join(target); - t!(fs::create_dir_all(&dst)); - let mut src = builder.sysroot_libdir(compiler, target).to_path_buf(); - src.pop(); // Remove the trailing /lib folder from the sysroot_libdir - builder.cp_filtered(&src, &dst, &|path| { - if let Some(name) = path.file_name().and_then(|s| s.to_str()) { - if name == builder.config.rust_codegen_backends_dir.as_str() { - return false - } - if name == "bin" { - return false - } - if name.contains("LLVM") { - return false - } - } - true - }); + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, &target, &image, &stamp); let mut cmd = rust_installer(builder); cmd.arg("generate") @@ -723,7 +720,73 @@ impl Step for Std { let _time = timeit(builder); builder.run(&mut cmd); builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target)) + archive + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDev { + pub compiler: Compiler, + pub target: Interned, +} + +impl Step for RustcDev { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("rustc-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDev { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let compiler = self.compiler; + let target = self.target; + + let name = pkgname(builder, "rustc-dev"); + let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target)); + if skip_host_target_lib(builder, compiler) { + return archive; + } + + builder.ensure(compile::Rustc { compiler, target }); + + let image = tmpdir(builder).join(format!("{}-{}-image", name, target)); + let _ = fs::remove_dir_all(&image); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, &target, &image, &stamp); + + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=Rust-is-ready-to-develop.") + .arg("--image-dir").arg(&image) + .arg("--work-dir").arg(&tmpdir(builder)) + .arg("--output-dir").arg(&distdir(builder)) + .arg(format!("--package-name={}-{}", name, target)) + .arg(format!("--component-name=rustc-dev-{}", target)) + .arg("--legacy-manifest-dirs=rustlib,cargo"); + + builder.info(&format!("Dist rustc-dev stage{} ({} -> {})", + compiler.stage, &compiler.host, target)); + let _time = timeit(builder); + builder.run(&mut cmd); + builder.remove_dir(&image); + archive } } diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index f41e7dd17ede6..97e758f9b8238 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -399,6 +399,7 @@ impl Builder { fn add_packages_to(&mut self, manifest: &mut Manifest) { let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets); package("rustc", HOSTS); + package("rustc-dev", HOSTS); package("cargo", HOSTS); package("rust-mingw", MINGW); package("rust-std", TARGETS); @@ -481,6 +482,15 @@ impl Builder { components.push(host_component("rust-mingw")); } + // The compiler libraries are not stable for end users, but `rustc-dev` was only recently + // split out of `rust-std`. We'll include it by default as a transition for nightly users, + // but ship it as an optional component on the beta and stable channels. + if self.rust_release == "nightly" { + components.push(host_component("rustc-dev")); + } else { + extensions.push(host_component("rustc-dev")); + } + // Tools are always present in the manifest, // but might be marked as unavailable if they weren't built. extensions.extend(vec![ @@ -498,6 +508,11 @@ impl Builder { .filter(|&&target| target != host) .map(|target| Component::from_str("rust-std", target)) ); + extensions.extend( + HOSTS.iter() + .filter(|&&target| target != host) + .map(|target| Component::from_str("rustc-dev", target)) + ); extensions.push(Component::from_str("rust-src", "*")); // If the components/extensions don't actually exist for this From 4c906dc84ec78fad35405d5c0f2b2d58f9f26288 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Mon, 7 Oct 2019 15:49:51 -0700 Subject: [PATCH 2/3] Add rustc-dev to nightly default and complete profiles --- src/tools/build-manifest/src/main.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 97e758f9b8238..c0d2deab2f8ba 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -427,6 +427,13 @@ impl Builder { "rls-preview", "rust-src", "llvm-tools-preview", "lldb-preview", "rust-analysis", "miri-preview" ]); + + // The compiler libraries are not stable for end users, but `rustc-dev` was only recently + // split out of `rust-std`. We'll include it by default as a transition for nightly users. + if self.rust_release == "nightly" { + self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]); + self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); + } } fn add_renames_to(&self, manifest: &mut Manifest) { @@ -549,6 +556,14 @@ impl Builder { dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect()); } + fn extend_profile(&mut self, + profile_name: &str, + dst: &mut BTreeMap>, + pkgs: &[&str]) { + dst.get_mut(profile_name).expect("existing profile") + .extend(pkgs.iter().map(|s| (*s).to_owned())); + } + fn package(&mut self, pkgname: &str, dst: &mut BTreeMap, From 7ccf492ae616b4d06eab283ab604938fd234415a Mon Sep 17 00:00:00 2001 From: Jethro Beekman Date: Sat, 12 Oct 2019 16:01:59 +0200 Subject: [PATCH 3/3] Package non-rust objects --- src/bootstrap/check.rs | 4 +++ src/bootstrap/compile.rs | 63 ++++++++++++++++++++++++++-------------- src/bootstrap/lib.rs | 1 + 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index cadb9a7e441f2..df1c72575846b 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -55,6 +55,7 @@ impl Step for Std { cargo, args(builder.kind), &libstd_stamp(builder, compiler, target), + vec![], true); let libdir = builder.sysroot_libdir(compiler, target); @@ -103,6 +104,7 @@ impl Step for Rustc { cargo, args(builder.kind), &librustc_stamp(builder, compiler, target), + vec![], true); let libdir = builder.sysroot_libdir(compiler, target); @@ -155,6 +157,7 @@ impl Step for CodegenBackend { cargo, args(builder.kind), &codegen_backend_stamp(builder, compiler, target, backend), + vec![], true); } } @@ -199,6 +202,7 @@ impl Step for Rustdoc { cargo, args(builder.kind), &rustdoc_stamp(builder, compiler, target), + vec![], true); let libdir = builder.sysroot_libdir(compiler, target); diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 6ea32edfb208b..d48927f9bf78f 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -69,7 +69,7 @@ impl Step for Std { return; } - builder.ensure(StartupObjects { compiler, target }); + let mut target_deps = builder.ensure(StartupObjects { compiler, target }); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); if compiler_to_use != compiler { @@ -91,7 +91,7 @@ impl Step for Std { return; } - copy_third_party_objects(builder, &compiler, target); + target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter()); let mut cargo = builder.cargo(compiler, Mode::Std, target, "build"); std_cargo(builder, &compiler, target, &mut cargo); @@ -102,6 +102,7 @@ impl Step for Std { cargo, vec![], &libstd_stamp(builder, compiler, target), + target_deps, false); builder.ensure(StdLink { @@ -113,9 +114,22 @@ impl Step for Std { } /// Copies third pary objects needed by various targets. -fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned) { +fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned) + -> Vec +{ let libdir = builder.sysroot_libdir(*compiler, target); + let mut target_deps = vec![]; + + let mut copy_and_stamp = |sourcedir: &Path, name: &str| { + let target = libdir.join(name); + builder.copy( + &sourcedir.join(name), + &target, + ); + target_deps.push(target); + }; + // Copies the crt(1,i,n).o startup objects // // Since musl supports fully static linking, we can cross link for it even @@ -123,19 +137,13 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: // files. As those shipped with glibc won't work, copy the ones provided by // musl so we have them on linux-gnu hosts. if target.contains("musl") { + let srcdir = builder.musl_root(target).unwrap().join("lib"); for &obj in &["crt1.o", "crti.o", "crtn.o"] { - builder.copy( - &builder.musl_root(target).unwrap().join("lib").join(obj), - &libdir.join(obj), - ); + copy_and_stamp(&srcdir, obj); } } else if target.ends_with("-wasi") { - for &obj in &["crt1.o"] { - builder.copy( - &builder.wasi_root(target).unwrap().join("lib/wasm32-wasi").join(obj), - &libdir.join(obj), - ); - } + let srcdir = builder.wasi_root(target).unwrap().join("lib/wasm32-wasi"); + copy_and_stamp(&srcdir, "crt1.o"); } // Copies libunwind.a compiled to be linked wit x86_64-fortanix-unknown-sgx. @@ -145,11 +153,11 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: // which is provided by std for this target. if target == "x86_64-fortanix-unknown-sgx" { let src_path_env = "X86_FORTANIX_SGX_LIBS"; - let obj = "libunwind.a"; let src = env::var(src_path_env).expect(&format!("{} not found in env", src_path_env)); - let src = Path::new(&src).join(obj); - builder.copy(&src, &libdir.join(obj)); + copy_and_stamp(Path::new(&src), "libunwind.a"); } + + target_deps } /// Configure cargo to compile the standard library, adding appropriate env vars @@ -307,7 +315,7 @@ pub struct StartupObjects { } impl Step for StartupObjects { - type Output = (); + type Output = Vec; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/rtstartup") @@ -326,13 +334,15 @@ impl Step for StartupObjects { /// They don't require any library support as they're just plain old object /// files, so we just use the nightly snapshot compiler to always build them (as /// no other compilers are guaranteed to be available). - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> Vec { let for_compiler = self.compiler; let target = self.target; if !target.contains("windows-gnu") { - return + return vec![] } + let mut target_deps = vec![]; + let src_dir = &builder.src.join("src/rtstartup"); let dst_dir = &builder.native_dir(target).join("rtstartup"); let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); @@ -351,7 +361,9 @@ impl Step for StartupObjects { .arg(src_file)); } - builder.copy(dst_file, &sysroot_dir.join(file.to_string() + ".o")); + let target = sysroot_dir.join(file.to_string() + ".o"); + builder.copy(dst_file, &target); + target_deps.push(target); } for obj in ["crt2.o", "dllcrt2.o"].iter() { @@ -359,8 +371,12 @@ impl Step for StartupObjects { builder.cc(target), target, obj); - builder.copy(&src, &sysroot_dir.join(obj)); + let target = sysroot_dir.join(obj); + builder.copy(&src, &target); + target_deps.push(target); } + + target_deps } } @@ -438,6 +454,7 @@ impl Step for Rustc { cargo, vec![], &librustc_stamp(builder, compiler, target), + vec![], false); builder.ensure(RustcLink { @@ -586,7 +603,7 @@ impl Step for CodegenBackend { let tmp_stamp = out_dir.join(".tmp.stamp"); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, false); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false); if builder.config.dry_run { return; } @@ -954,6 +971,7 @@ pub fn run_cargo(builder: &Builder<'_>, cargo: Cargo, tail_args: Vec, stamp: &Path, + additional_target_deps: Vec, is_check: bool) -> Vec { @@ -1070,6 +1088,7 @@ pub fn run_cargo(builder: &Builder<'_>, deps.push((path_to_add.into(), false)); } + deps.extend(additional_target_deps.into_iter().map(|d| (d, false))); deps.sort(); let mut new_contents = Vec::new(); for (dep, proc_macro) in deps.iter() { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 9203a558f6465..ad0bdd0f425f1 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1144,6 +1144,7 @@ impl Build { pub fn copy(&self, src: &Path, dst: &Path) { if self.config.dry_run { return; } self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst)); + if src == dst { return; } let _ = fs::remove_file(&dst); let metadata = t!(src.symlink_metadata()); if metadata.file_type().is_symlink() {