From c5b9ea02dbfd4987ebec403a3db4d70fa028ad0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Fri, 24 Jun 2022 17:04:40 +0200 Subject: [PATCH] start of bake implementation --- xtask/src/bake.json | 37 ++++++ xtask/src/bake_docker_image.rs | 229 ++++++++++++++++++++++++++++++++ xtask/src/build_docker_image.rs | 68 ++++++---- xtask/src/main.rs | 9 ++ 4 files changed, 320 insertions(+), 23 deletions(-) create mode 100644 xtask/src/bake.json create mode 100644 xtask/src/bake_docker_image.rs diff --git a/xtask/src/bake.json b/xtask/src/bake.json new file mode 100644 index 000000000..27e02426c --- /dev/null +++ b/xtask/src/bake.json @@ -0,0 +1,37 @@ +{ + "group": { + "default": { + "targets": ["x86_64-unknown-linux-gnu", "x86_64-unknown-netbsd"] + } + }, + "target": { + "x86_64-unknown-linux-gnu": { + "args": {}, + "cache-from": [ + "type=registry,ref=ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main" + ], + "context": null, + "dockerfile": "/home/emil/workspace/cross/docker/Dockerfile.x86_64-unknown-linux-gnu", + "labels": { + "org.cross-rs.for-cross-target": "x86_64-unknown-linux-gnu" + }, + "no-cache": false, + "platforms": ["linux/amd64"], + "tags": ["ghcr.io/cross-rs/x86_64-unknown-linux-gnu:local"] + }, + "x86_64-unknown-netbsd": { + "args": {}, + "cache-from": [ + "type=registry,ref=ghcr.io/cross-rs/x86_64-unknown-netbsd:main" + ], + "context": null, + "dockerfile": "/home/emil/workspace/cross/docker/Dockerfile.x86_64-unknown-netbsd", + "labels": { + "org.cross-rs.for-cross-target": "x86_64-unknown-netbsd" + }, + "no-cache": false, + "platforms": ["linux/amd64"], + "tags": ["ghcr.io/cross-rs/x86_64-unknown-netbsd:local"] + } + } +} diff --git a/xtask/src/bake_docker_image.rs b/xtask/src/bake_docker_image.rs new file mode 100644 index 000000000..272e6e30a --- /dev/null +++ b/xtask/src/bake_docker_image.rs @@ -0,0 +1,229 @@ +use std::{collections::BTreeMap, path::Path}; + +use clap::Args; +use cross::docker; +use serde::Serialize; + +use crate::build_docker_image::{get_tags, locate_dockerfile}; + +#[derive(Args, Debug)] +pub struct BakeDockerImage { + #[clap(long, hide = true, env = "GITHUB_REF_TYPE")] + ref_type: Option, + #[clap(long, hide = true, env = "GITHUB_REF_NAME")] + ref_name: Option, + #[clap(long, hide = true, env = "IS_LATEST")] + is_latest: Option, + /// Specify a tag to use instead of the derived one, eg `local` + #[clap(long)] + tag: Option, + /// Repository name for image. + #[clap(long, default_value = docker::CROSS_IMAGE)] + repository: String, + /// Newline separated labels + #[clap(long, env = "LABELS")] + labels: Option, + /// Provide verbose diagnostic output. + #[clap(short, long)] + pub verbose: bool, + /// Print but do not execute the build commands. + #[clap(long)] + dry_run: bool, + /// Force a push when `--push` is set, but not `--tag` + #[clap(long, hide = true)] + force: bool, + /// Push build to registry. + #[clap(short, long)] + push: bool, + /// Set output to /dev/null + #[clap(short, long)] + no_output: bool, + /// Docker build progress output type. + #[clap( + long, + value_parser = clap::builder::PossibleValuesParser::new(["auto", "plain", "tty"]), + default_value = "auto" + )] + progress: String, + /// Use a bake build + #[clap(long)] + bake: bool, + /// Do not load from cache when building the image. + #[clap(long)] + no_cache: bool, + /// Continue building images even if an image fails to build. + #[clap(long)] + no_fastfail: bool, + /// Container engine (such as docker or podman). + #[clap(long)] + pub engine: Option, + /// If no target list is provided, parse list from CI. + #[clap(long)] + from_ci: bool, + /// Additional build arguments to pass to Docker. + #[clap(long)] + build_arg: Vec, + /// Targets to build for + #[clap()] + targets: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct BakeTarget { + context: Option, + dockerfile: String, + tags: Vec, + labels: BTreeMap, + platforms: Vec, + args: BTreeMap, + no_cache: bool, + cache_from: Vec, +} + +pub fn bake_docker_image( + BakeDockerImage { + ref_type, + ref_name, + is_latest, + tag: tag_override, + repository, + labels, + verbose, + push, + no_output, + no_cache, + from_ci, + build_arg, + mut targets, + .. + }: BakeDockerImage, + engine: &docker::Engine, +) -> cross::Result<()> { + let metadata = cross::cargo_metadata_with_args( + Some(Path::new(env!("CARGO_MANIFEST_DIR"))), + None, + verbose, + )? + .ok_or_else(|| eyre::eyre!("could not find cross workspace and its current version"))?; + let version = metadata + .get_package("cross") + .expect("cross expected in workspace") + .version + .clone(); + if targets.is_empty() { + if from_ci { + targets = crate::util::get_matrix() + .iter() + .filter(|m| m.os.starts_with("ubuntu")) + .map(|m| m.to_image_target()) + .collect(); + } else { + targets = walkdir::WalkDir::new(metadata.workspace_root.join("docker")) + .max_depth(1) + .contents_first(true) + .into_iter() + .filter_map(|e| e.ok().filter(|f| f.file_type().is_file())) + .filter_map(|f| { + f.file_name() + .to_string_lossy() + .strip_prefix("Dockerfile.") + .map(ToOwned::to_owned) + .map(|s| s.parse().unwrap()) + }) + .collect(); + } + } + let docker_root = metadata.workspace_root.join("docker"); + let cross_toolchains_root = docker_root.join("cross-toolchains").join("docker"); + let targets = targets + .into_iter() + .map(|t| locate_dockerfile(t, &docker_root, &cross_toolchains_root)) + .collect::>>()?; + let build_args = build_arg + .into_iter() + .map(|s| -> cross::Result<_> { + let s = s + .split_once('=') + .ok_or_else(|| eyre::eyre!("invalid build arg `{s}`"))?; + Ok((s.0.to_string(), s.1.to_string())) + }) + .collect::>>()?; + let labels = labels + .into_iter() + .map(|s| -> cross::Result<_> { + let s = s + .split_once('=') + .ok_or_else(|| eyre::eyre!("invalid label `{s}`"))?; + Ok((s.0.to_string(), s.1.to_string())) + }) + .collect::>>()?; + let targets = targets + .into_iter() + .map(|(target, dockerfile)| -> cross::Result<_> { + Ok(( + target.to_string().replace('.', "-"), + BakeTarget { + context: None, + dockerfile, + tags: get_tags( + &target, + &repository, + &version, + is_latest.unwrap_or_default(), + ref_type.as_deref(), + ref_name.as_deref(), + tag_override.as_deref(), + )?, + labels: { + let mut labels = labels.clone(); + labels.insert( + format!("{}.for-cross-target", cross::CROSS_LABEL_DOMAIN), + target.triplet.clone(), + ); + labels + }, + platforms: vec!["linux/amd64".to_string()], + args: build_args.clone(), + no_cache, + cache_from: if !no_cache { + vec![format!( + "type=registry,ref={}", + target.image_name(&repository, "main") + )] + } else { + vec![] + }, + }, + )) + }) + .collect::>>()?; + + let mut docker_bake = docker::command(engine); + docker_bake.args(&["buildx", "bake"]); + docker_bake.current_dir(&docker_root); + + if push { + docker_bake.arg("--push"); + } else if no_output { + docker_bake.args(&["--set", "*.output=type=tar,dest=/dev/null"]); + } else { + docker_bake.arg("--load"); + } + let defaults = targets + .iter() + .map(|(t, _)| t.clone()) + .collect::>(); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "group": { + "default": { + "targets": defaults, + } + }, + "target": targets, + }))? + ); + todo!() +} diff --git a/xtask/src/build_docker_image.rs b/xtask/src/build_docker_image.rs index c82a93ea7..2f0ba5042 100644 --- a/xtask/src/build_docker_image.rs +++ b/xtask/src/build_docker_image.rs @@ -5,6 +5,8 @@ use color_eyre::Section; use cross::{docker, CommandExt, ToUtf8}; use std::fmt::Write; +use crate::util::ImageTarget; + #[derive(Args, Debug)] pub struct BuildDockerImage { #[clap(long, hide = true, env = "GITHUB_REF_TYPE")] @@ -44,6 +46,9 @@ pub struct BuildDockerImage { default_value = "auto" )] progress: String, + /// Use a bake build + #[clap(long)] + bake: bool, /// Do not load from cache when building the image. #[clap(long)] no_cache: bool, @@ -64,7 +69,7 @@ pub struct BuildDockerImage { targets: Vec, } -fn locate_dockerfile( +pub fn locate_dockerfile( target: crate::ImageTarget, docker_root: &Path, cross_toolchain_root: &Path, @@ -147,6 +152,9 @@ pub fn build_docker_image( .collect::>>()?; let mut results = vec![]; + if push && tag_override.is_none() { + panic!("Refusing to push without tag or branch. Specify a repository and tag with `--repository --tag `") + } for (target, dockerfile) in &targets { if gha && targets.len() > 1 { println!("::group::Build {target}"); @@ -163,28 +171,15 @@ pub fn build_docker_image( docker_build.arg("--load"); } - let mut tags = vec![]; - - match (ref_type.as_deref(), ref_name.as_deref()) { - (Some(ref_type), Some(ref_name)) => tags.extend(determine_image_name( - target, - &repository, - ref_type, - ref_name, - is_latest.unwrap_or_default(), - &version, - )?), - _ => { - if push && tag_override.is_none() { - panic!("Refusing to push without tag or branch. Specify a repository and tag with `--repository --tag `") - } - tags.push(target.image_name(&repository, "local")); - } - } - - if let Some(ref tag) = tag_override { - tags = vec![target.image_name(&repository, tag)]; - } + let tags: Vec<_> = get_tags( + target, + &repository, + &version, + is_latest.unwrap_or_default(), + ref_type.as_deref(), + ref_name.as_deref(), + tag_override.as_deref(), + )?; docker_build.arg("--pull"); if no_cache { @@ -282,6 +277,33 @@ pub fn build_docker_image( Ok(()) } +pub fn get_tags( + target: &ImageTarget, + repository: &str, + version: &str, + is_latest: bool, + ref_type: Option<&str>, + ref_name: Option<&str>, + tag_override: Option<&str>, +) -> cross::Result> { + if let Some(tag) = tag_override { + return Ok(vec![target.image_name(repository, tag)]); + } + + let mut tags = vec![]; + + match (ref_type, ref_name) { + (Some(ref_type), Some(ref_name)) => tags.extend(determine_image_name( + target, repository, ref_type, ref_name, is_latest, version, + )?), + _ => { + tags.push(target.image_name(repository, "local")); + } + } + + Ok(tags) +} + pub fn determine_image_name( target: &crate::ImageTarget, repository: &str, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 858c759e3..cfa990825 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,5 +1,6 @@ #![deny(missing_debug_implementations, rust_2018_idioms)] +pub mod bake_docker_image; pub mod build_docker_image; pub mod ci; pub mod hooks; @@ -12,6 +13,7 @@ use clap::{CommandFactory, Parser, Subcommand}; use cross::docker; use util::ImageTarget; +use self::bake_docker_image::BakeDockerImage; use self::build_docker_image::BuildDockerImage; use self::hooks::{Check, Test}; use self::install_git_hooks::InstallGitHooks; @@ -40,6 +42,8 @@ enum Commands { TargetInfo(TargetInfo), /// Build a docker image from file. BuildDockerImage(BuildDockerImage), + /// Bake docker image(s) from file. + BakeDockerImage(BakeDockerImage), /// Install git development hooks. InstallGitHooks(InstallGitHooks), /// Run code formatting checks and lints. @@ -72,6 +76,11 @@ pub fn main() -> cross::Result<()> { let engine = get_container_engine(args.engine.as_deref(), args.verbose)?; build_docker_image::build_docker_image(args, &engine)?; } + Commands::BakeDockerImage(args) => { + let engine = get_container_engine(args.engine.as_deref(), args.verbose)?; + bake_docker_image::bake_docker_image(args, &engine)?; + } + Commands::InstallGitHooks(args) => { install_git_hooks::install_git_hooks(args)?; }