build_manifest: use prebuilt build-manifest from tarballs

Cargo.toml

@@ -5,6 +5,8 @@ authors = ["Alex Crichton <alex@alexcrichton.com>"]
 license = "MIT OR Apache-2.0"
 edition = "2018"
 
+build = "build.rs"
+
 [dependencies]
 curl = "0.4"
 flate2 = "1"

build.rs

@@ -0,0 +1,10 @@
+fn main() {
+    // Prevent a rebuild every time a non-Rust file is changed.
+    println!("cargo:rerun-if-changed=build.rs");
+
+    // Provide the current target as environment variable.
+    println!(
+        "cargo:rustc-env=TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

local/run.sh

@@ -13,6 +13,7 @@ RUSTC_DEFAULT_BRANCH="master"
 DOWNLOAD_BASE="https://ci-artifacts.rust-lang.org/rustc-builds"
 # Rustup components to download for each target we want to release.
 DOWNLOAD_COMPONENTS=(
+    "build-manifest"
     "cargo"
     "rust"
     "rust-docs"

prod/Dockerfile

@@ -22,7 +22,7 @@ ENV PATH=/root/.cargo/bin:$PATH
 # caching, and it works by copying the Cargo.{toml,lock} with dummy source code
 # and doing a full build with it.
 WORKDIR /tmp/source
-COPY Cargo.lock Cargo.toml /tmp/source/
+COPY Cargo.lock Cargo.toml build.rs /tmp/source/
 RUN mkdir -p /tmp/source/src && \
     echo "fn main() {}" > /tmp/source/src/main.rs
 RUN cargo fetch

src/build_manifest.rs

@@ -0,0 +1,85 @@
+use crate::{config::Channel, Context};
+use anyhow::{Context as _, Error};
+use std::{
+    fs::File,
+    io::BufReader,
+    path::{Path, PathBuf},
+    process::Command,
+};
+use tar::Archive;
+use tempfile::NamedTempFile;
+use xz2::read::XzDecoder;
+
+pub(crate) struct BuildManifest<'a> {
+    builder: &'a Context,
+    tarball_name: String,
+    tarball_path: PathBuf,
+}
+
+impl<'a> BuildManifest<'a> {
+    pub(crate) fn new(builder: &'a Context) -> Self {
+        // Precalculate paths used later.
+        let release = match builder.config.channel {
+            Channel::Stable => builder.current_version.clone().unwrap(),
+            channel => channel.to_string(),
+        };
+        let tarball_name = format!("build-manifest-{}-{}", release, crate::TARGET);
+        let tarball_path = builder.dl_dir().join(format!("{}.tar.xz", tarball_name));
+
+        Self {
+            builder,
+            tarball_name,
+            tarball_path,
+        }
+    }
+
+    pub(crate) fn exists(&self) -> bool {
+        self.tarball_path.is_file()
+    }
+
+    pub(crate) fn run(&self) -> Result<(), Error> {
+        let config = &self.builder.config;
+        let bin = self
+            .extract()
+            .context("failed to extract build-manifest from the tarball")?;
+
+        println!("running build-manifest...");
+        let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir);
+        // build-manifest <input-dir> <output-dir> <date> <upload-addr> <channel>
+        let status = Command::new(bin.path())
+            .arg(self.builder.dl_dir())
+            .arg(self.builder.dl_dir())
+            .arg(&self.builder.date)
+            .arg(upload_addr)
+            .arg(config.channel.to_string())
+            .status()
+            .context("failed to execute build-manifest")?;
+
+        if status.success() {
+            return Ok(());
+        } else {
+            anyhow::bail!("build-manifest failed with status {:?}", status);
+        }
+    }
+
+    fn extract(&self) -> Result<NamedTempFile, Error> {
+        let binary_path = Path::new(&self.tarball_name)
+            .join("build-manifest")
+            .join("bin")
+            .join("build-manifest");
+
+        let tarball_file = BufReader::new(File::open(&self.tarball_path)?);
+        let mut tarball = Archive::new(XzDecoder::new(tarball_file));
+
+        let bin = NamedTempFile::new()?;
+        tarball
+            .entries()?
+            .filter_map(|e| e.ok())
+            .filter(|e| e.path().ok().as_deref() == Some(&binary_path))
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("missing build-manifest binary inside the tarball"))?
+            .unpack(bin.path())?;
+
+        Ok(bin)
+    }
+}

src/main.rs

@@ -1,3 +1,4 @@
+mod build_manifest;
 mod config;
 mod sign;
 
@@ -9,14 +10,18 @@ use std::process::Command;
 use std::time::Instant;
 
 use anyhow::Error;
+use build_manifest::BuildManifest;
+use chrono::Utc;
 use curl::easy::Easy;
 use fs2::FileExt;
 use rayon::prelude::*;
-use chrono::Utc;
 use sign::Signer;
+use xz2::read::XzDecoder;
 
 use crate::config::{Channel, Config};
 
+const TARGET: &str = env!("TARGET");
+
 struct Context {
     work: PathBuf,
     handle: Easy,
@@ -51,7 +56,6 @@ impl Context {
 
     fn run(&mut self) -> Result<(), Error> {
         let _lock = self.lock()?;
-        self.update_repo()?;
         self.do_release()?;
 
         Ok(())
@@ -73,7 +77,7 @@ impl Context {
 
     /// Update the rust repository we have cached, either cloning a fresh one or
     /// fetching remote references
-    fn update_repo(&mut self) -> Result<(), Error> {
+    fn legacy_update_repo(&mut self) -> Result<(), Error> {
         // Clone/update the repo
         let dir = self.rust_dir();
         if dir.is_dir() {
@@ -168,34 +172,23 @@ impl Context {
 
         self.assert_all_components_present()?;
 
-        // Ok we've now determined that a release needs to be done. Let's
-        // configure rust, build a manifest and sign the artifacts we just downloaded, and upload the
-        // signatures and manifest to the CI bucket.
-
-        self.configure_rust(&rev)?;
+        // Ok we've now determined that a release needs to be done.
 
-        let is_legacy_build_manifest = !self
-            .rust_dir()
-            .join("src/tools/build-manifest/src/manifest.rs")
-            .exists();
-        println!("is legacy build-manifest: {}", is_legacy_build_manifest);
-
-        if is_legacy_build_manifest {
-            self.sign_artifacts()?;
-        } else {
-            self.build_manifest()?;
-        }
+        let build_manifest = BuildManifest::new(self);
+        if build_manifest.exists() {
+            // Generate the channel manifest
+            build_manifest.run()?;
 
-        // Merge all the signatures with the download files, and then sync that
-        // whole dir up to the release archives
-        for file in self.build_dir().join("build/dist/").read_dir()? {
-            let file = file?;
-            fs::copy(file.path(), self.dl_dir().join(file.file_name()))?;
-        }
-
-        if !is_legacy_build_manifest {
+            // Generate checksums and sign all the files we're about to ship.
             let signer = Signer::new(&self.config)?;
             signer.sign_directory(&self.dl_dir())?;
+        } else {
+            // For releases using the legacy build-manifest, we need to clone the rustc monorepo
+            // and invoke `./x.py dist hash-and-sign` in it. This won't be needed after 1.48.0 is
+            // out in the stable channel.
+            self.legacy_update_repo()?;
+            self.legacy_configure_rust(&rev)?;
+            self.legacy_sign_artifacts()?;
         }
 
         self.publish_archive()?;
@@ -211,7 +204,7 @@ impl Context {
         Ok(())
     }
 
-    fn configure_rust(&mut self, rev: &str) -> Result<(), Error> {
+    fn legacy_configure_rust(&mut self, rev: &str) -> Result<(), Error> {
         let build = self.build_dir();
         // Only delete the dist artifacts when running the tool locally, to avoid rebuilding
         // bootstrap over and over again.
@@ -428,7 +421,7 @@ upload-addr = \"{}/{}\"
                     println!("recompressing {}...", gz_path.display());
 
                     let xz = File::open(xz_path)?;
-                    let mut xz = xz2::read::XzDecoder::new(xz);
+                    let mut xz = XzDecoder::new(xz);
                     let gz = File::create(gz_path)?;
                     let mut gz = flate2::write::GzEncoder::new(gz, compression_level);
                     io::copy(&mut xz, &mut gz)?;
@@ -448,20 +441,22 @@ upload-addr = \"{}/{}\"
     }
 
     /// Create manifest and sign the artifacts.
-    fn sign_artifacts(&mut self) -> Result<(), Error> {
+    fn legacy_sign_artifacts(&mut self) -> Result<(), Error> {
         let build = self.build_dir();
         // This calls `src/tools/build-manifest` from the rustc repo.
         run(Command::new(self.rust_dir().join("x.py"))
             .current_dir(&build)
             .arg("dist")
-            .arg("hash-and-sign"))
-    }
+            .arg("hash-and-sign"))?;
 
-    fn build_manifest(&mut self) -> Result<(), Error> {
-        run(Command::new(self.rust_dir().join("x.py"))
-            .current_dir(&self.build_dir())
-            .arg("run")
-            .arg("src/tools/build-manifest"))
+        // Merge all the signatures with the download files, and then sync that
+        // whole dir up to the release archives
+        for file in self.build_dir().join("build/dist/").read_dir()? {
+            let file = file?;
+            fs::copy(file.path(), self.dl_dir().join(file.file_name()))?;
+        }
+
+        Ok(())
     }
 
     fn publish_archive(&mut self) -> Result<(), Error> {