diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index fcbde9c438cb5..0a79d09b27fed 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -19,6 +19,7 @@ use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::compile; use crate::config::TargetSelection; +use crate::tarball::{OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, timeit}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; @@ -36,10 +37,6 @@ pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { builder.out.join("tmp/dist") } -fn rust_installer(builder: &Builder<'_>) -> Command { - builder.tool_cmd(Tool::RustInstaller) -} - fn missing_tool(tool_name: &str, skip: bool) { if skip { println!("Unable to build {}, skipping dist", tool_name) @@ -54,7 +51,7 @@ pub struct Docs { } impl Step for Docs { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -66,48 +63,20 @@ impl Step for Docs { } /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - - let name = pkgname(builder, "rust-docs"); - if !builder.config.docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } - builder.default_doc(None); - builder.info(&format!("Dist docs ({})", host)); - let _time = timeit(builder); + let dest = "share/doc/rust/html"; - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - - let dst = image.join("share/doc/rust/html"); - t!(fs::create_dir_all(&dst)); - let src = builder.doc_out(host); - builder.cp_r(&src, &dst); - builder.install(&builder.src.join("src/doc/robots.txt"), &dst, 0o644); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html"); - builder.run(&mut cmd); - builder.remove_dir(&image); - - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) } } @@ -117,7 +86,7 @@ pub struct RustcDocs { } impl Step for RustcDocs { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -129,47 +98,17 @@ impl Step for RustcDocs { } /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - - let name = pkgname(builder, "rustc-docs"); - if !builder.config.compiler_docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } - builder.default_doc(None); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - - let dst = image.join("share/doc/rust/html/rustc"); - t!(fs::create_dir_all(&dst)); - let src = builder.compiler_doc_out(host); - builder.cp_r(&src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rustc-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rustc-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html/rustc"); - - builder.info(&format!("Dist compiler docs ({})", host)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) } } @@ -345,41 +284,20 @@ impl Step for Mingw { /// without any extra installed software (e.g., we bundle gcc, libraries, etc). fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - if !host.contains("pc-windows-gnu") { return None; } - builder.info(&format!("Dist mingw ({})", host)); - let _time = timeit(builder); - let name = pkgname(builder, "rust-mingw"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image)); + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); // The first argument is a "temporary directory" which is just // thrown away (this contains the runtime DLLs included in the rustc package // above) and the second argument is where to place all the MinGW components // (which is what we want). - make_win_dist(&tmpdir(builder), &image, host, &builder); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-MinGW") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-MinGW-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-mingw") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - builder.run(&mut cmd); - t!(fs::remove_dir_all(&image)); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple))) + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); + + Some(tarball.generate()) } } @@ -407,30 +325,10 @@ impl Step for Rustc { let compiler = self.compiler; let host = self.compiler.host; - let name = pkgname(builder, "rustc"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host.triple)); - let _ = fs::remove_dir_all(&overlay); + let tarball = Tarball::new(builder, "rustc", &host.triple); // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, &image); - - // Prepare the overlay which is part of the tarball but won't actually be - // installed - let cp = |file: &str| { - builder.install(&builder.src.join(file), &overlay, 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - // tiny morsel of metadata is used by rust-packaging - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); - } + prepare_image(builder, compiler, tarball.image_dir()); // On MinGW we've got a few runtime DLL dependencies that we need to // include. The first argument to this script is where to put these DLLs @@ -443,38 +341,11 @@ impl Step for Rustc { // install will *also* include the rust-mingw package, which also needs // licenses, so to be safe we just include it here in all MinGW packages. if host.contains("pc-windows-gnu") { - make_win_dist(&image, &tmpdir(builder), host, builder); - - let dst = image.join("share/doc"); - t!(fs::create_dir_all(&dst)); - builder.cp_r(&builder.src.join("src/etc/third-party"), &dst); + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); } - // Finally, wrap everything up in a nice tarball! - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - builder.remove_dir(&overlay); - - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return tarball.generate(); fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { let host = compiler.host; @@ -684,7 +555,7 @@ pub struct Std { } impl Step for Std { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -702,46 +573,24 @@ impl Step for Std { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - let name = pkgname(builder, "rust-std"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Std { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=std-is-standing-at-the-ready.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-std-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder - .info(&format!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); + + Some(tarball.generate()) } } @@ -752,7 +601,7 @@ pub struct RustcDev { } impl Step for RustcDev { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -771,60 +620,36 @@ impl Step for RustcDev { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - - let name = pkgname(builder, "rustc-dev"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Rustc { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); - - // Copy compiler sources. - let dst_src = image.join("lib/rustlib/rustc-src/rust"); - t!(fs::create_dir_all(&dst_src)); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); - let src_files = ["Cargo.lock"]; + let src_files = &["Cargo.lock"]; // This is the reduced set of paths which will become the rustc-dev component // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs(builder, &builder.src, &["compiler"], &[], &dst_src); - for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); } - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-develop.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rustc-dev-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!( - "Dist rustc-dev stage{} ({} -> {})", - compiler.stage, &compiler.host, target - )); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + Some(tarball.generate()) } } @@ -835,7 +660,7 @@ pub struct Analysis { } impl Step for Analysis { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -858,52 +683,26 @@ impl Step for Analysis { } /// Creates a tarball of save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let name = pkgname(builder, "rust-analysis"); - if compiler.host != builder.config.build { - return distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); + return None; } builder.ensure(compile::Std { compiler, target }); - - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let src = builder .stage_out(compiler, Mode::Std) .join(target.triple) .join(builder.cargo_dir()) - .join("deps"); + .join("deps") + .join("save-analysis"); - let image_src = src.join("save-analysis"); - let dst = image.join("lib/rustlib").join(target.triple).join("analysis"); - t!(fs::create_dir_all(&dst)); - builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst)); - builder.cp_r(&image_src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=save-analysis-saved.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-analysis-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info("Dist analysis"); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) } } @@ -1011,9 +810,7 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> PathBuf { - let name = pkgname(builder, "rust-src"); - let image = tmpdir(builder).join(format!("{}-image", name)); - let _ = fs::remove_dir_all(&image); + let tarball = Tarball::new_targetless(builder, "rust-src"); // A lot of tools expect the rust-src component to be entirely in this directory, so if you // change that (e.g. by adding another directory `lib/rustlib/src/foo` or @@ -1022,8 +819,7 @@ impl Step for Src { // // NOTE: if you update the paths here, you also should update the "virtual" path // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = image.join("lib/rustlib/src/rust"); - t!(fs::create_dir_all(&dst_src)); + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); let src_files = ["Cargo.lock"]; // This is the reduced set of paths which will become the rust-src component @@ -1043,28 +839,7 @@ impl Step for Src { builder.copy(&builder.src.join(file), &dst_src.join(file)); } - // Create source tarball in rust-installer format - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Awesome-Source.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}", name)) - .arg("--component-name=rust-src") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info("Dist src"); - let _time = timeit(builder); - builder.run(&mut cmd); - - builder.remove_dir(&image); - distdir(builder).join(&format!("{}.tar.gz", name)) + tarball.generate() } } @@ -1088,11 +863,8 @@ impl Step for PlainSourceTarball { /// Creates the plain source tarball fn run(self, builder: &Builder<'_>) -> PathBuf { - // Make sure that the root folder of tarball has the correct name - let plain_name = format!("{}-src", pkgname(builder, "rustc")); - let plain_dst_src = tmpdir(builder).join(&plain_name); - let _ = fs::remove_dir_all(&plain_dst_src); - t!(fs::create_dir_all(&plain_dst_src)); + let tarball = Tarball::new(builder, "rustc", "src"); + let plain_dst_src = tarball.image_dir(); // This is the set of root paths which will become part of the source package let src_files = [ @@ -1135,28 +907,7 @@ impl Step for PlainSourceTarball { builder.run(&mut cmd); } - // Create plain source tarball - let plain_name = format!("rustc-{}-src", builder.rust_package_vers()); - let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name)); - tarball.set_extension(""); // strip .gz - tarball.set_extension(""); // strip .tar - if let Some(dir) = tarball.parent() { - builder.create_dir(&dir); - } - builder.info("running installer"); - let mut cmd = rust_installer(builder); - cmd.arg("tarball") - .arg("--input") - .arg(&plain_name) - .arg("--output") - .arg(&tarball) - .arg("--work-dir=.") - .current_dir(tmpdir(builder)); - - builder.info("Create plain source tarball"); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(&format!("{}.tar.gz", plain_name)) + tarball.bare() } } @@ -1212,72 +963,28 @@ impl Step for Cargo { let compiler = self.compiler; let target = self.target; + let cargo = builder.ensure(tool::Cargo { compiler, target }); let src = builder.src.join("src/tools/cargo"); let etc = src.join("src/etc"); - let release_num = builder.release_num("cargo"); - let name = pkgname(builder, "cargo"); - let version = builder.cargo_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("cargo-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); // Prepare the image directory - builder.create_dir(&image.join("share/zsh/site-functions")); - builder.create_dir(&image.join("etc/bash_completion.d")); - let cargo = builder.ensure(tool::Cargo { compiler, target }); - builder.install(&cargo, &image.join("bin"), 0o755); + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { let dirent = dirent.expect("read dir entry"); if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") { - builder.install(&dirent.path(), &image.join("libexec"), 0o755); + tarball.add_file(&dirent.path(), "libexec", 0o755); } } - for man in t!(etc.join("man").read_dir()) { - let man = t!(man); - builder.install(&man.path(), &image.join("share/man/man1"), 0o644); - } - builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644); - builder.copy(&etc.join("cargo.bashcomp.sh"), &image.join("etc/bash_completion.d/cargo")); - let doc = image.join("share/doc/cargo"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("cargo-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--component-name=cargo") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + + tarball.generate() } } @@ -1311,19 +1018,6 @@ impl Step for Rls { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/rls"); - let release_num = builder.release_num("rls"); - let name = pkgname(builder, "rls"); - let version = builder.rls_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rls-image"); - drop(fs::remove_dir_all(&image)); - t!(fs::create_dir_all(&image)); - - // Prepare the image directory - // We expect RLS to build, because we've exited this step above if tool - // state for RLS isn't testing. let rls = builder .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1331,43 +1025,12 @@ impl Step for Rls { None })?; - builder.install(&rls, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rls"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rls-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=RLS-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rls-preview"); - - builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) } } @@ -1407,60 +1070,16 @@ impl Step for RustAnalyzer { return None; } - let src = builder.src.join("src/tools/rust-analyzer"); - let release_num = builder.release_num("rust-analyzer/crates/rust-analyzer"); - let name = pkgname(builder, "rust-analyzer"); - let version = builder.rust_analyzer_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-analyzer-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect rust-analyer to always build, as it doesn't depend on rustc internals - // and doesn't have associated toolstate. let rust_analyzer = builder .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() }) .expect("rust-analyzer always builds"); - builder.install(&rust_analyzer, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rust-analyzer"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rust-analyzer-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-analyzer-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-analyzer-preview"); - - builder.info(&format!("Dist rust-analyzer stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) } } @@ -1494,16 +1113,6 @@ impl Step for Clippy { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/clippy"); - let release_num = builder.release_num("clippy"); - let name = pkgname(builder, "clippy"); - let version = builder.clippy_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("clippy-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool // state for clippy isn't testing. @@ -1514,44 +1123,13 @@ impl Step for Clippy { .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) .expect("clippy expected to build - essential tool"); - builder.install(&clippy, &image.join("bin"), 0o755); - builder.install(&cargoclippy, &image.join("bin"), 0o755); - let doc = image.join("share/doc/clippy"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("clippy-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=clippy-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=clippy-preview"); - - builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + tarball.generate() } } @@ -1585,19 +1163,6 @@ impl Step for Miri { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/miri"); - let release_num = builder.release_num("miri"); - let name = pkgname(builder, "miri"); - let version = builder.miri_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("miri-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect miri to build, because we've exited this step above if tool - // state for miri isn't testing. let miri = builder .ensure(tool::Miri { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1611,44 +1176,13 @@ impl Step for Miri { None })?; - builder.install(&miri, &image.join("bin"), 0o755); - builder.install(&cargomiri, &image.join("bin"), 0o755); - let doc = image.join("share/doc/miri"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("miri-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=miri-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=miri-preview"); - - builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) } } @@ -1681,17 +1215,6 @@ impl Step for Rustfmt { let compiler = self.compiler; let target = self.target; - let src = builder.src.join("src/tools/rustfmt"); - let release_num = builder.release_num("rustfmt"); - let name = pkgname(builder, "rustfmt"); - let version = builder.rustfmt_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rustfmt-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory let rustfmt = builder .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1705,44 +1228,13 @@ impl Step for Rustfmt { None })?; - builder.install(&rustfmt, &image.join("bin"), 0o755); - builder.install(&cargofmt, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rustfmt"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rustfmt-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rustfmt-ready-to-fmt.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rustfmt-preview"); - - builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) } } @@ -1791,24 +1283,14 @@ impl Step for Extended { let analysis_installer = builder.ensure(Analysis { compiler, target }); let docs_installer = builder.ensure(Docs { host: target }); - let std_installer = - builder.ensure(Std { compiler: builder.compiler(stage, target), target }); + let std_installer = builder.ensure(Std { compiler, target }); - let tmp = tmpdir(builder); - let overlay = tmp.join("extended-overlay"); let etc = builder.src.join("src/etc/installer"); - let work = tmp.join("work"); - - let _ = fs::remove_dir_all(&overlay); - builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644); - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); + + // Avoid producing tarballs during a dry run. + if builder.config.dry_run { + return; } - builder.install(&etc.join("README.md"), &overlay, 0o644); // When rust-std package split from rustc, we needed to ensure that during // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering @@ -1823,39 +1305,22 @@ impl Step for Extended { tarballs.extend(miri_installer.clone()); tarballs.extend(rustfmt_installer.clone()); tarballs.extend(llvm_tools_installer); - tarballs.push(analysis_installer); - tarballs.push(std_installer); - if builder.config.docs { + if let Some(analysis_installer) = analysis_installer { + tarballs.push(analysis_installer); + } + tarballs.push(std_installer.expect("missing std")); + if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); } if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } - let mut input_tarballs = tarballs[0].as_os_str().to_owned(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(tarball); - } - builder.info("building combined installer"); - let mut cmd = rust_installer(builder); - cmd.arg("combine") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--work-dir") - .arg(&work) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--input-tarballs") - .arg(input_tarballs) - .arg("--non-installed-overlay") - .arg(&overlay); - let time = timeit(&builder); - builder.run(&mut cmd); - drop(time); + let mut tarball = Tarball::new(builder, "rust", &target.triple); + let work = tarball.persist_work_dir(); + tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); let mut license = String::new(); license += &builder.read(&builder.src.join("COPYRIGHT")); @@ -2428,58 +1893,25 @@ impl Step for LlvmTools { } } - builder.info(&format!("Dist LlvmTools ({})", target)); - let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "llvm-tools"); - - let tmp = tmpdir(builder); - let image = tmp.join("llvm-tools-image"); - drop(fs::remove_dir_all(&image)); + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); // Prepare the image directory let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = image.join("lib/rustlib").join(&*target.triple).join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); for tool in LLVM_TOOLS { let exe = src_bindir.join(exe(tool, target)); - builder.install(&exe, &dst_bindir, 0o755); + tarball.add_file(&exe, &dst_bindir, 0o755); } // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm_target(builder, target, &image); - - // Prepare the overlay - let overlay = tmp.join("llvm-tools-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.llvm_tools_vers()); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=llvm-tools-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=llvm-tools-preview"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + maybe_install_llvm_target(builder, target, tarball.image_dir()); + + Some(tarball.generate()) } } @@ -2515,70 +1947,35 @@ impl Step for RustDev { } } - builder.info(&format!("Dist RustDev ({})", target)); - let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "rust-dev"); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-dev-image"); - drop(fs::remove_dir_all(&image)); - - // Prepare the image directory - let dst_bindir = image.join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); let src_bindir = builder.llvm_out(target).join("bin"); - let install_bin = - |name| builder.install(&src_bindir.join(exe(name, target)), &dst_bindir, 0o755); - install_bin("llvm-config"); - install_bin("llvm-ar"); - install_bin("llvm-objdump"); - install_bin("llvm-profdata"); - install_bin("llvm-bcanalyzer"); - install_bin("llvm-cov"); - install_bin("llvm-dwp"); - builder.install(&builder.llvm_filecheck(target), &dst_bindir, 0o755); + for bin in &[ + "llvm-config", + "llvm-ar", + "llvm-objdump", + "llvm-profdata", + "llvm-bcanalyzer", + "llvm-cov", + "llvm-dwp", + ] { + tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); + } + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); // Copy the include directory as well; needed mostly to build // librustc_llvm properly (e.g., llvm-config.h is in here). But also // just broadly useful to be able to link against the bundled LLVM. - builder.cp_r(&builder.llvm_out(target).join("include"), &image.join("include")); + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm(builder, target, &image.join("lib")); - - // Prepare the overlay - let overlay = tmp.join("rust-dev-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.rust_version()); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-dev-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-dev"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + maybe_install_llvm(builder, target, &tarball.image_dir().join("lib")); + + Some(tarball.generate()) } } @@ -2607,45 +2004,9 @@ impl Step for BuildManifest { fn run(self, builder: &Builder<'_>) -> PathBuf { let build_manifest = builder.tool_exe(Tool::BuildManifest); - let name = pkgname(builder, "build-manifest"); - let tmp = tmpdir(builder); - - // Prepare the image. - let image = tmp.join("build-manifest-image"); - let image_bin = image.join("bin"); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image_bin)); - builder.install(&build_manifest, &image_bin, 0o755); - - // Prepare the overlay. - let overlay = tmp.join("build-manifest-overlay"); - let _ = fs::remove_dir_all(&overlay); - builder.create_dir(&overlay); - builder.create(&overlay.join("version"), &builder.rust_version()); - for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { - builder.install(&builder.src.join(file), &overlay, 0o644); - } - - // Create the final tarball. - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=build-manifest installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, self.target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=build-manifest"); - - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple)) + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() } } @@ -2673,47 +2034,10 @@ impl Step for ReproducibleArtifacts { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let name = pkgname(builder, "reproducible-artifacts"); - let tmp = tmpdir(builder); - - // Prepare the image. - let image = tmp.join("reproducible-artifacts-image"); - let _ = fs::remove_dir_all(&image); - - if let Some(path) = &builder.config.rust_profile_use { - builder.install(std::path::Path::new(path), &image, 0o644); - } else { - return None; - } - - // Prepare the overlay. - let overlay = tmp.join("reproducible-artifacts-overlay"); - let _ = fs::remove_dir_all(&overlay); - builder.create_dir(&overlay); - builder.create(&overlay.join("version"), &builder.rust_version()); - for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { - builder.install(&builder.src.join(file), &overlay, 0o644); - } + let path = builder.config.rust_profile_use.as_ref()?; - // Create the final tarball. - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=reproducible-artifacts installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, self.target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=reproducible-artifacts"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))) + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + tarball.add_file(path, ".", 0o644); + Some(tarball.generate()) } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index ece9bdc7a6499..a47ddfbcc1f18 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -142,6 +142,7 @@ mod native; mod run; mod sanity; mod setup; +mod tarball; mod test; mod tool; mod toolstate; @@ -1068,10 +1069,6 @@ impl Build { self.package_vers(&self.version) } - fn llvm_tools_vers(&self) -> String { - self.rust_version() - } - fn llvm_link_tools_dynamically(&self, target: TargetSelection) -> bool { target.contains("linux-gnu") || target.contains("apple-darwin") } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs new file mode 100644 index 0000000000000..5d73a655427b2 --- /dev/null +++ b/src/bootstrap/tarball.rs @@ -0,0 +1,298 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use build_helper::t; + +use crate::builder::Builder; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, + Cargo, + Clippy, + Miri, + Rustfmt, + RLS, + RustAnalyzer, +} + +impl OverlayKind { + fn legal_and_readme(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], + OverlayKind::RLS => &[ + "src/tools/rls/README.md", + "src/tools/rls/LICENSE-APACHE", + "src/tools/rls/LICENSE-MIT", + ], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } + OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: Option, + product_name: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + + include_target_in_component_name: bool, + is_preview: bool, + delete_temp_dir: bool, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { + let pkgname = crate::dist::pkgname(builder, component); + + let mut temp_dir = builder.out.join("tmp").join("tarball"); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + + Self { + builder, + + pkgname, + component: component.into(), + target, + product_name: "Rust".into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + + include_target_in_component_name: false, + is_preview: false, + delete_temp_dir: true, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef, + destdir: impl AsRef, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); + } + + pub(crate) fn persist_work_dir(&mut self) -> PathBuf { + self.delete_temp_dir = false; + self.temp_dir.clone() + } + + pub(crate) fn generate(self) -> PathBuf { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg(format!("--component-name={}", &component_name)); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn combine(self, tarballs: &[PathBuf]) { + let mut input_tarballs = tarballs[0].as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(tarball); + } + + self.run(|this, cmd| { + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); + }); + } + + pub(crate) fn bare(self) -> PathBuf { + self.run(|this, cmd| { + cmd.arg("tarball") + .arg("--input") + .arg(&this.image_dir) + .arg("--output") + .arg(crate::dist::distdir(this.builder).join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(crate::dist::distdir(self.builder)); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> PathBuf { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); + if let Some(sha) = self.builder.rust_sha() { + self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); + } + for file in self.overlay.legal_and_readme() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + + let package_name = self.package_name(); + self.builder.info(&format!("Dist {}", package_name)); + let _time = crate::util::timeit(self.builder); + + build_cli(&self, &mut cmd); + cmd.arg("--work-dir").arg(&self.temp_dir); + self.builder.run(&mut cmd); + if self.delete_temp_dir { + t!(std::fs::remove_dir_all(&self.temp_dir)); + } + + crate::dist::distdir(self.builder).join(format!("{}.tar.gz", package_name)) + } +}