diff --git a/Cargo.toml b/Cargo.toml index ac77c17..2e6c758 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,8 @@ authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" edition = "2018" +build = "build.rs" + [dependencies] curl = "0.4" flate2 = "1" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..9f0e8c8 --- /dev/null +++ b/build.rs @@ -0,0 +1,7 @@ +fn main() { + // Prevent a rebuild every time a non-Rust file is changed. + println!("cargo:rerun-if-changed=build.rs"); + + // Provide the current target as environment variable. + println!("cargo:rustc-env=TARGET={}", std::env::var("TARGET").unwrap()); +} diff --git a/local/run.sh b/local/run.sh index 108aa9a..d515ee1 100755 --- a/local/run.sh +++ b/local/run.sh @@ -13,6 +13,7 @@ RUSTC_DEFAULT_BRANCH="master" DOWNLOAD_BASE="https://ci-artifacts.rust-lang.org/rustc-builds" # Rustup components to download for each target we want to release. DOWNLOAD_COMPONENTS=( + "build-manifest" "cargo" "rust" "rust-docs" diff --git a/src/build_manifest.rs b/src/build_manifest.rs new file mode 100644 index 0000000..f6b7c99 --- /dev/null +++ b/src/build_manifest.rs @@ -0,0 +1,85 @@ +use crate::{config::Channel, Context}; +use anyhow::{Context as _, Error}; +use std::{ + fs::File, + io::BufReader, + path::{Path, PathBuf}, + process::Command, +}; +use tar::Archive; +use tempfile::NamedTempFile; +use xz2::read::XzDecoder; + +pub(crate) struct BuildManifest<'a> { + builder: &'a Context, + tarball_name: String, + tarball_path: PathBuf, +} + +impl<'a> BuildManifest<'a> { + pub(crate) fn new(builder: &'a Context) -> Self { + // Precalculate paths used later. + let release = match builder.config.channel { + Channel::Stable => builder.current_version.clone().unwrap(), + channel => channel.to_string(), + }; + let tarball_name = format!("build-manifest-{}-{}", release, crate::TARGET); + let tarball_path = builder.dl_dir().join(format!("{}.tar.xz", tarball_name)); + + Self { + builder, + tarball_name, + tarball_path, + } + } + + pub(crate) fn exists(&self) -> bool { + self.tarball_path.is_file() + } + + pub(crate) fn run(&self) -> Result<(), Error> { + let config = &self.builder.config; + let bin = self + .extract() + .context("failed to extract build-manifest from the tarball")?; + + println!("running build-manifest..."); + let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir); + // build-manifest + let status = Command::new(bin.path()) + .arg(self.builder.dl_dir()) + .arg(self.builder.dl_dir()) + .arg(&self.builder.date) + .arg(upload_addr) + .arg(config.channel.to_string()) + .status() + .context("failed to execute build-manifest")?; + + if status.success() { + return Ok(()); + } else { + anyhow::bail!("build-manifest failed with status {:?}", status); + } + } + + fn extract(&self) -> Result { + let binary_path = Path::new(&self.tarball_name) + .join("build-manifest") + .join("bin") + .join("build-manifest"); + + let tarball_file = BufReader::new(File::open(&self.tarball_path)?); + let mut tarball = Archive::new(XzDecoder::new(tarball_file)); + + let bin = NamedTempFile::new()?; + tarball + .entries()? + .filter_map(|e| e.ok()) + .filter(|e| e.path().ok().as_deref() == Some(&binary_path)) + .next() + .ok_or_else(|| anyhow::anyhow!("missing build-manifest binary inside the tarball"))? + .unpack(bin.path())?; + + Ok(bin) + } +} diff --git a/src/main.rs b/src/main.rs index a1e7c56..5ed320e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod build_manifest; mod config; mod sign; @@ -9,14 +10,18 @@ use std::process::Command; use std::time::Instant; use anyhow::Error; +use build_manifest::BuildManifest; +use chrono::Utc; use curl::easy::Easy; use fs2::FileExt; use rayon::prelude::*; -use chrono::Utc; use sign::Signer; +use xz2::read::XzDecoder; use crate::config::{Channel, Config}; +const TARGET: &str = env!("TARGET"); + struct Context { work: PathBuf, handle: Easy, @@ -51,7 +56,6 @@ impl Context { fn run(&mut self) -> Result<(), Error> { let _lock = self.lock()?; - self.update_repo()?; self.do_release()?; Ok(()) @@ -73,7 +77,7 @@ impl Context { /// Update the rust repository we have cached, either cloning a fresh one or /// fetching remote references - fn update_repo(&mut self) -> Result<(), Error> { + fn legacy_update_repo(&mut self) -> Result<(), Error> { // Clone/update the repo let dir = self.rust_dir(); if dir.is_dir() { @@ -168,34 +172,23 @@ impl Context { self.assert_all_components_present()?; - // Ok we've now determined that a release needs to be done. Let's - // configure rust, build a manifest and sign the artifacts we just downloaded, and upload the - // signatures and manifest to the CI bucket. - - self.configure_rust(&rev)?; + // Ok we've now determined that a release needs to be done. - let is_legacy_build_manifest = !self - .rust_dir() - .join("src/tools/build-manifest/src/manifest.rs") - .exists(); - println!("is legacy build-manifest: {}", is_legacy_build_manifest); - - if is_legacy_build_manifest { - self.sign_artifacts()?; - } else { - self.build_manifest()?; - } + let build_manifest = BuildManifest::new(self); + if build_manifest.exists() { + // Generate the channel manifest + build_manifest.run()?; - // Merge all the signatures with the download files, and then sync that - // whole dir up to the release archives - for file in self.build_dir().join("build/dist/").read_dir()? { - let file = file?; - fs::copy(file.path(), self.dl_dir().join(file.file_name()))?; - } - - if !is_legacy_build_manifest { + // Generate checksums and sign all the files we're about to ship. let signer = Signer::new(&self.config)?; signer.sign_directory(&self.dl_dir())?; + } else { + // For releases using the legacy build-manifest, we need to clone the rustc monorepo + // and invoke `./x.py dist hash-and-sign` in it. This won't be needed after 1.48.0 is + // out in the stable channel. + self.legacy_update_repo()?; + self.legacy_configure_rust(&rev)?; + self.legacy_sign_artifacts()?; } self.publish_archive()?; @@ -211,7 +204,7 @@ impl Context { Ok(()) } - fn configure_rust(&mut self, rev: &str) -> Result<(), Error> { + fn legacy_configure_rust(&mut self, rev: &str) -> Result<(), Error> { let build = self.build_dir(); // Only delete the dist artifacts when running the tool locally, to avoid rebuilding // bootstrap over and over again. @@ -428,7 +421,7 @@ upload-addr = \"{}/{}\" println!("recompressing {}...", gz_path.display()); let xz = File::open(xz_path)?; - let mut xz = xz2::read::XzDecoder::new(xz); + let mut xz = XzDecoder::new(xz); let gz = File::create(gz_path)?; let mut gz = flate2::write::GzEncoder::new(gz, compression_level); io::copy(&mut xz, &mut gz)?; @@ -448,20 +441,22 @@ upload-addr = \"{}/{}\" } /// Create manifest and sign the artifacts. - fn sign_artifacts(&mut self) -> Result<(), Error> { + fn legacy_sign_artifacts(&mut self) -> Result<(), Error> { let build = self.build_dir(); // This calls `src/tools/build-manifest` from the rustc repo. run(Command::new(self.rust_dir().join("x.py")) .current_dir(&build) .arg("dist") - .arg("hash-and-sign")) - } + .arg("hash-and-sign"))?; - fn build_manifest(&mut self) -> Result<(), Error> { - run(Command::new(self.rust_dir().join("x.py")) - .current_dir(&self.build_dir()) - .arg("run") - .arg("src/tools/build-manifest")) + // Merge all the signatures with the download files, and then sync that + // whole dir up to the release archives + for file in self.build_dir().join("build/dist/").read_dir()? { + let file = file?; + fs::copy(file.path(), self.dl_dir().join(file.file_name()))?; + } + + Ok(()) } fn publish_archive(&mut self) -> Result<(), Error> {