diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..f4067fc82 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,16 @@ +# This workflow runs when commits are pushed to main or a branch starting with +# "version-". It checks if any packages require a new release, and if so, +# creates the crates.io release and git tag. +name: Release +on: + push: + branches: + - main + - version-* +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - cargo xtask auto-release diff --git a/Cargo.lock b/Cargo.lock index 019adb44f..d08665ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,44 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.83" @@ -160,6 +198,26 @@ dependencies = [ "libc", ] +[[package]] +name = "crates-index" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1f3e3ef6d547bbf1213b3dabbf0b01a500fbd0924abf818b563a4bc2c85296c" +dependencies = [ + "hex", + "home", + "http", + "memchr", + "rustc-hash", + "semver", + "serde", + "serde_derive", + "serde_json", + "smol_str", + "thiserror", + "toml", +] + [[package]] name = "crc" version = "3.0.1" @@ -210,6 +268,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.6" @@ -259,6 +323,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -307,12 +377,47 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "http" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f95b9abcae896730d42b78e09c155ed4ddf82c07b4de772c64aee5b2d8b7c150" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "idna" version = "0.4.0" @@ -323,6 +428,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itertools" version = "0.11.0" @@ -538,6 +653,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.21" @@ -598,6 +719,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +dependencies = [ + "serde", +] + [[package]] name = "serde" version = "1.0.192" @@ -638,6 +768,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "sha2" version = "0.10.8" @@ -649,6 +788,15 @@ dependencies = [ "digest", ] +[[package]] +name = "smol_str" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" +dependencies = [ + "serde", +] + [[package]] name = "spin" version = "0.9.8" @@ -751,6 +899,40 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "trybuild" version = "1.0.85" @@ -883,6 +1065,7 @@ checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3" dependencies = [ "base64", "flate2", + "http", "log", "once_cell", "rustls", @@ -1027,6 +1210,15 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winnow" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +dependencies = [ + "memchr", +] + [[package]] name = "wyz" version = "0.5.1" @@ -1050,7 +1242,9 @@ name = "xtask" version = "0.0.0" dependencies = [ "anyhow", + "cargo_metadata", "clap", + "crates-index", "fatfs", "fs-err", "heck", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 32435fad1..58852746b 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -7,7 +7,9 @@ edition.workspace = true [dependencies] anyhow = "1.0.51" +cargo_metadata = "0.18.1" clap = { version = "4.4.0", default-features = false, features = ["derive", "help", "usage", "std"] } +crates-index = "2.3.0" fatfs = { version = "0.3.6", default-features = false, features = ["alloc", "std"] } fs-err = "2.6.0" heck = "0.4.0" @@ -24,5 +26,5 @@ sha2 = "0.10.6" syn = { version = "2.0.0", features = ["full"] } tar = "0.4.38" tempfile = "3.6.0" +ureq = { version = "2.8.0", features = ["http-interop"] } walkdir = "2.4.0" -ureq = "2.8.0" diff --git a/xtask/src/cargo.rs b/xtask/src/cargo.rs index adcdb4881..2fc782995 100644 --- a/xtask/src/cargo.rs +++ b/xtask/src/cargo.rs @@ -16,7 +16,8 @@ pub enum Package { } impl Package { - fn as_str(self) -> &'static str { + /// Get package name. + pub fn name(self) -> &'static str { match self { Self::Uefi => "uefi", Self::UefiApp => "uefi_app", @@ -28,12 +29,12 @@ impl Package { } } - /// All published packages. + /// All published packages, in the order that publishing should occur. pub fn published() -> Vec { vec![ - Self::Uefi, - Self::UefiMacros, Self::UefiRaw, + Self::UefiMacros, + Self::Uefi, Self::UefiServices, ] } @@ -304,7 +305,7 @@ impl Cargo { bail!("packages cannot be empty"); } for package in &self.packages { - cmd.args(["--package", package.as_str()]); + cmd.args(["--package", package.name()]); } if !self.features.is_empty() { diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 8cba0feeb..f3804e7ea 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -8,6 +8,7 @@ mod opt; mod pipe; mod platform; mod qemu; +mod release; mod tpm; mod util; @@ -314,5 +315,6 @@ fn main() -> Result<()> { Action::Run(qemu_opt) => run_vm_tests(qemu_opt), Action::Test(test_opt) => run_host_tests(test_opt), Action::Fmt(fmt_opt) => run_fmt_project(fmt_opt), + Action::AutoRelease(_) => release::auto_release(), } } diff --git a/xtask/src/opt.rs b/xtask/src/opt.rs index af3f06250..98457d225 100644 --- a/xtask/src/opt.rs +++ b/xtask/src/opt.rs @@ -74,6 +74,7 @@ pub enum Action { Run(QemuOpt), Test(TestOpt), Fmt(FmtOpt), + AutoRelease(AutoReleaseOpt), } /// Build all the uefi packages. @@ -202,3 +203,9 @@ pub struct FmtOpt { #[clap(long, action)] pub check: bool, } + +/// Run the auto-release process. +/// +/// This is run by the `release` workflow. +#[derive(Debug, Parser)] +pub struct AutoReleaseOpt {} diff --git a/xtask/src/release.rs b/xtask/src/release.rs new file mode 100644 index 000000000..5a6546ea9 --- /dev/null +++ b/xtask/src/release.rs @@ -0,0 +1,205 @@ +use crate::cargo::Package; +use crate::util; +use anyhow::{Context, Result}; +use cargo_metadata::{Metadata, MetadataCommand}; +use crates_index::SparseIndex; +use std::env; +use std::process::Command; + +/// Entry point for the auto-release process. This is intended to be run from a +/// Github Actions workflow, see `.github/workflows/release.yml`. +pub fn auto_release() -> Result<()> { + let commit_sha = get_commit_sha()?; + let commit_message_body = get_commit_message_body(&commit_sha)?; + + if !commit_message_body.starts_with("release: ") { + println!("{commit_sha} does not contain the release trigger"); + return Ok(()); + } + + fetch_git_tags()?; + + let local_metadata = get_local_package_metadata()?; + let mut index = SparseIndex::new_cargo_default()?; + + for package in Package::published() { + auto_release_package(package, &local_metadata, &mut index, &commit_sha)?; + } + + Ok(()) +} + +/// Release a single package, if needed. +/// +/// This publishes to crates.io if the corresponding version does not already +/// exist there, and also pushes a new git tag if one doesn't exist yet. +fn auto_release_package( + package: Package, + local_metadata: &Metadata, + index: &mut SparseIndex, + commit_sha: &str, +) -> Result<()> { + let local_version = get_local_package_version(package, local_metadata)?; + println!("local version of {} is {local_version}", package.name()); + + // Create the remote git tag if it doesn't exist. + let tag = get_git_tag_name(package, &local_version); + if does_git_tag_exist(&tag)? { + println!("git tag {tag} already exists"); + } else { + make_and_push_git_tag(&tag, commit_sha)?; + } + + // Create the crates.io release if it doesn't exist. + if does_crates_io_release_exist(package, &local_version, index)? { + println!( + "{}-{local_version} has already been published", + package.name() + ); + } else { + publish_package(package)?; + } + + Ok(()) +} + +/// Get the commit to operate on from the `GITHUB_SHA` env var. When running in +/// Github Actions, this will be set to the SHA of the merge commit that was +/// pushed to the branch. +fn get_commit_sha() -> Result { + let commit_var_name = "GITHUB_SHA"; + env::var(commit_var_name).context(format!("failed to get env var {commit_var_name}")) +} + +/// Create a git command with the given args. +fn get_git_command(args: [&str; N]) -> Command { + let mut cmd = Command::new("git"); + cmd.args(args); + cmd +} + +/// Get the body of the commit message for the given commit. +fn get_commit_message_body(commit_sha: &str) -> Result { + let cmd = get_git_command([ + "log", + "-1", + // Only get the body of the commit message. + "--format=format:%b", + commit_sha, + ]); + let output = util::get_cmd_stdout(cmd)?; + String::from_utf8(output).context("commit message is not utf-8") +} + +/// Use the `cargo_metadata` crate to get local info about packages in the +/// workspace. +fn get_local_package_metadata() -> Result { + let mut cmd = MetadataCommand::new(); + // Ignore deps, we only need local packages. + cmd.no_deps(); + Ok(cmd.exec()?) +} + +/// Fetch git tags from the remote. +fn fetch_git_tags() -> Result<()> { + let cmd = get_git_command(["fetch", "--tags"]); + util::run_cmd(cmd) +} + +/// Format a package version as a git tag. +fn get_git_tag_name(package: Package, local_version: &str) -> String { + format!("{}-v{}", package.name(), local_version) +} + +/// Check if a git tag exists locally. +/// +/// All git tags were fetched at the start of auto-release, so checking locally +/// is sufficient. +fn does_git_tag_exist(tag: &str) -> Result { + let cmd = get_git_command(["tag", "--list", tag]); + let output = util::get_cmd_stdout(cmd)?; + let output = String::from_utf8(output).context("git tag is not utf-8")?; + + Ok(output.lines().any(|line| line == tag)) +} + +/// Create a git tag locally and push it. +fn make_and_push_git_tag(tag: &str, commit_sha: &str) -> Result<()> { + // Create the tag. + let cmd = get_git_command(["tag", tag, commit_sha]); + util::run_cmd(cmd)?; + + // Push it. + let cmd = get_git_command(["push", "--tags"]); + util::run_cmd(cmd) +} + +/// Update the local crates.io cache. +/// +/// Based on https://github.com/frewsxcv/rust-crates-index/blob/HEAD/examples/sparse_http_ureq.rs +fn update_index(index: &mut SparseIndex, package: Package) -> Result<()> { + let crate_name = package.name(); + + println!("fetching updates for {}", package.name()); + let request: ureq::Request = index.make_cache_request(crate_name).unwrap().into(); + let response = request.call()?; + + index.parse_cache_response(crate_name, response.into(), true)?; + + Ok(()) +} + +/// Check if a new release of `package` should be published. +fn does_crates_io_release_exist( + package: Package, + local_version: &str, + index: &mut SparseIndex, +) -> Result { + let remote_versions = get_remote_package_versions(package, index)?; + if remote_versions.contains(&local_version.to_string()) { + return Ok(true); + } + + Ok(false) +} + +/// Get the local version of `package`. +fn get_local_package_version(package: Package, local_metadata: &Metadata) -> Result { + let metadata = local_metadata + .packages + .iter() + .find(|pm| pm.name == package.name()) + .context(format!( + "failed to find {} in local metadata", + package.name() + ))?; + Ok(metadata.version.to_string()) +} + +/// Get all remote versions of `package`. +fn get_remote_package_versions(package: Package, index: &mut SparseIndex) -> Result> { + // The local cache may be out of date, fetch updates from the remote. + update_index(index, package)?; + + let cr = index.crate_from_cache(package.name())?; + + Ok(cr + .versions() + .iter() + .map(|v| v.version().to_string()) + .collect()) +} + +/// Publish `package` to crates.io. +fn publish_package(package: Package) -> Result<()> { + let mut cmd = Command::new("cargo"); + cmd.args([ + "publish", + package.name(), + // TODO: drop this + "--dry-run", + // TODO: add token + // "--token", + ]); + util::run_cmd(cmd) +} diff --git a/xtask/src/util.rs b/xtask/src/util.rs index 86a2da63b..97ae615fb 100644 --- a/xtask/src/util.rs +++ b/xtask/src/util.rs @@ -44,6 +44,19 @@ pub fn run_cmd(mut cmd: Command) -> Result<()> { } } +/// Print a `Command` and run it, then check that it completes +/// successfully. Return the command's stdout. +pub fn get_cmd_stdout(mut cmd: Command) -> Result> { + println!("run_cmd: '{}'", command_to_string(&cmd)); + + let output = cmd.output()?; + if output.status.success() { + Ok(output.stdout) + } else { + bail!("command failed: {}", output.status); + } +} + #[cfg(test)] mod tests { use super::*;