From f82e6a1807f86a1415317a802f44ec9346f8f282 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Tue, 16 Jul 2024 19:57:11 +0000 Subject: [PATCH 1/3] fix bug in call to 'docker image inspect' The --format string erroneously surrounded the output of `docker image inspect` in quotes, causing the JSON parser to read the output as a string instead of an object. --- Cargo.lock | 1 + tools/oci-cli-wrapper/Cargo.toml | 1 + tools/oci-cli-wrapper/src/cli.rs | 20 ++++++++++++++++++++ tools/oci-cli-wrapper/src/docker.rs | 8 +------- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdbee8f34..078465d55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2167,6 +2167,7 @@ name = "oci-cli-wrapper" version = "0.1.0" dependencies = [ "async-trait", + "log", "regex", "serde", "serde_json", diff --git a/tools/oci-cli-wrapper/Cargo.toml b/tools/oci-cli-wrapper/Cargo.toml index 3072328f9..6f48e867a 100644 --- a/tools/oci-cli-wrapper/Cargo.toml +++ b/tools/oci-cli-wrapper/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] async-trait = "0.1" +log = "0.4" regex = "1" serde = { version = "1", features = ["derive"]} serde_json = "1" diff --git a/tools/oci-cli-wrapper/src/cli.rs b/tools/oci-cli-wrapper/src/cli.rs index b75f0cad6..fe99afa06 100644 --- a/tools/oci-cli-wrapper/src/cli.rs +++ b/tools/oci-cli-wrapper/src/cli.rs @@ -10,6 +10,14 @@ pub(crate) struct CommandLine { impl CommandLine { pub(crate) async fn output(&self, args: &[&str], error_msg: String) -> Result> { + log::debug!( + "Executing '{}' with args [{}]", + self.path.display(), + args.iter() + .map(|arg| format!("'{}'", arg)) + .collect::>() + .join(", ") + ); let output = Command::new(&self.path) .args(args) .output() @@ -23,10 +31,22 @@ impl CommandLine { args: args.iter().map(|x| x.to_string()).collect::>() } ); + log::debug!( + "stdout: {}", + String::from_utf8_lossy(&output.stdout).to_string() + ); Ok(output.stdout) } pub(crate) async fn spawn(&self, args: &[&str], error_msg: String) -> Result<()> { + log::debug!( + "Executing '{}' with args [{}]", + self.path.display(), + args.iter() + .map(|arg| format!("'{}'", arg)) + .collect::>() + .join(", ") + ); let status = Command::new(&self.path) .args(args) .spawn() diff --git a/tools/oci-cli-wrapper/src/docker.rs b/tools/oci-cli-wrapper/src/docker.rs index 27ff962dc..ed337b070 100644 --- a/tools/oci-cli-wrapper/src/docker.rs +++ b/tools/oci-cli-wrapper/src/docker.rs @@ -57,13 +57,7 @@ impl ImageTool for DockerCLI { let bytes = self .cli .output( - &[ - "image", - "inspect", - uri, - "--format", - "\"{{ json .Config }}\"", - ], + &["image", "inspect", uri, "--format", "{{ json .Config }}"], format!("failed to fetch image config from {}", uri), ) .await?; From 98ca177da88aee9b0d01385c95706e9610154a53 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Wed, 17 Jul 2024 00:16:10 +0000 Subject: [PATCH 2/3] always canonicalize image manifests Prior to canonicalizing image manifests, crane and docker would result in slightly different image digests due to non-semantic formatting differences. --- Cargo.lock | 1 + tools/oci-cli-wrapper/Cargo.toml | 1 + tools/oci-cli-wrapper/src/cli.rs | 1 + tools/oci-cli-wrapper/src/crane.rs | 5 +- tools/oci-cli-wrapper/src/docker.rs | 5 +- tools/oci-cli-wrapper/src/lib.rs | 158 +++++++++++++++++++----- tools/pubsys/src/kit/publish_kit/mod.rs | 9 +- twoliter/src/lock.rs | 45 +++---- 8 files changed, 154 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 078465d55..17dc5cc8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,6 +2168,7 @@ version = "0.1.0" dependencies = [ "async-trait", "log", + "olpc-cjson", "regex", "serde", "serde_json", diff --git a/tools/oci-cli-wrapper/Cargo.toml b/tools/oci-cli-wrapper/Cargo.toml index 6f48e867a..e39cea7bf 100644 --- a/tools/oci-cli-wrapper/Cargo.toml +++ b/tools/oci-cli-wrapper/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] async-trait = "0.1" log = "0.4" +olpc-cjson = "0.1" regex = "1" serde = { version = "1", features = ["derive"]} serde_json = "1" diff --git a/tools/oci-cli-wrapper/src/cli.rs b/tools/oci-cli-wrapper/src/cli.rs index fe99afa06..4c3606e87 100644 --- a/tools/oci-cli-wrapper/src/cli.rs +++ b/tools/oci-cli-wrapper/src/cli.rs @@ -4,6 +4,7 @@ use tokio::process::Command; use crate::{error, Result}; +#[derive(Debug)] pub(crate) struct CommandLine { pub(crate) path: PathBuf, } diff --git a/tools/oci-cli-wrapper/src/crane.rs b/tools/oci-cli-wrapper/src/crane.rs index 5e0a166d7..cb5177211 100644 --- a/tools/oci-cli-wrapper/src/crane.rs +++ b/tools/oci-cli-wrapper/src/crane.rs @@ -7,15 +7,16 @@ use tar::Archive as TarArchive; use tempfile::TempDir; use crate::{ - cli::CommandLine, error, ConfigView, DockerArchitecture, ImageTool, ImageView, Result, + cli::CommandLine, error, ConfigView, DockerArchitecture, ImageToolImpl, ImageView, Result, }; +#[derive(Debug)] pub struct CraneCLI { pub(crate) cli: CommandLine, } #[async_trait] -impl ImageTool for CraneCLI { +impl ImageToolImpl for CraneCLI { async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> { let archive_path = path.to_string_lossy(); self.cli diff --git a/tools/oci-cli-wrapper/src/docker.rs b/tools/oci-cli-wrapper/src/docker.rs index ed337b070..f2cf140e7 100644 --- a/tools/oci-cli-wrapper/src/docker.rs +++ b/tools/oci-cli-wrapper/src/docker.rs @@ -8,14 +8,15 @@ use tar::Archive; use tempfile::NamedTempFile; use crate::cli::CommandLine; -use crate::{error, ConfigView, DockerArchitecture, ImageTool, Result}; +use crate::{error, ConfigView, DockerArchitecture, ImageToolImpl, Result}; +#[derive(Debug)] pub struct DockerCLI { pub(crate) cli: CommandLine, } #[async_trait] -impl ImageTool for DockerCLI { +impl ImageToolImpl for DockerCLI { async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> { // First we pull the image to local daemon self.cli diff --git a/tools/oci-cli-wrapper/src/lib.rs b/tools/oci-cli-wrapper/src/lib.rs index bfc21b083..b673ec72d 100644 --- a/tools/oci-cli-wrapper/src/lib.rs +++ b/tools/oci-cli-wrapper/src/lib.rs @@ -12,13 +12,14 @@ //! metadata. In addition, in order to operate with OCI image format, the containerd-snapshotter //! feature has to be enabled in the docker daemon use std::fmt::{Display, Formatter}; -use std::{collections::HashMap, env, path::Path, rc::Rc}; +use std::{collections::HashMap, env, path::Path}; use async_trait::async_trait; use cli::CommandLine; use crane::CraneCLI; use docker::DockerCLI; -use serde::Deserialize; +use olpc_cjson::CanonicalFormatter; +use serde::{Deserialize, Serialize}; use snafu::ResultExt; use which::which; @@ -26,8 +27,122 @@ mod cli; mod crane; mod docker; +#[derive(Debug)] +pub struct ImageTool { + image_tool_impl: Box, +} + +impl ImageTool { + /// Uses the container tool specified by the given tool name. + /// + /// The specified tool must be present in the unix search path. + fn from_tool_name(tool_name: &str) -> Result { + let image_tool_impl: Box = match tool_name { + "docker" => Box::new(DockerCLI { + cli: CommandLine { + path: which("docker").context(error::NotFoundSnafu { name: "docker" })?, + }, + }), + tool @ ("crane" | "gcrane" | "krane") => Box::new(CraneCLI { + cli: CommandLine { + path: which(tool).context(error::NotFoundSnafu { name: tool })?, + }, + }), + _ => return error::UnsupportedSnafu { name: tool_name }.fail(), + }; + + Ok(Self { image_tool_impl }) + } + + /// Auto-selects the container tool based on unix search path. + /// + /// Uses `crane` if available, falling back to `docker` otherwise. + fn from_unix_search_path() -> Result { + let crane = which("krane").or(which("gcrane")).or(which("crane")); + let image_tool_impl: Box = if let Ok(path) = crane { + Box::new(CraneCLI { + cli: CommandLine { path }, + }) + } else { + Box::new(DockerCLI { + cli: CommandLine { + path: which("docker").context(error::NoneFoundSnafu)?, + }, + }) + }; + + Ok(Self { image_tool_impl }) + } + + /// Auto-select the container tool to use by environment variable + /// and-or auto detection. + /// + /// If TWOLITER_KIT_IMAGE_TOOL environment variable is set, uses that value. + /// Valid values are: + /// * docker + /// * crane | gcrane | krane + /// + /// Otherwise, searches $PATH, using `crane` if available and falling back to docker otherwise. + pub fn from_environment() -> Result { + if let Ok(name) = env::var("TWOLITER_KIT_IMAGE_TOOL") { + Self::from_tool_name(&name) + } else { + Self::from_unix_search_path() + } + } + + pub fn new(image_tool_impl: Box) -> Self { + Self { image_tool_impl } + } + + /// Pull an image archive to disk + pub async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> { + self.image_tool_impl.pull_oci_image(path, uri).await + } + + /// Fetch the image config + pub async fn get_config(&self, uri: &str) -> Result { + self.image_tool_impl.get_config(uri).await + } + + /// Fetch the manifest + pub async fn get_manifest(&self, uri: &str) -> Result> { + let manifest_bytes = self.image_tool_impl.get_manifest(uri).await?; + let manifest_object: serde_json::Value = + serde_json::from_slice(&manifest_bytes).context(error::ManifestDeserializeSnafu)?; + + let mut canonicalized_manifest = Vec::new(); + let mut ser = serde_json::Serializer::with_formatter( + &mut canonicalized_manifest, + CanonicalFormatter::new(), + ); + + manifest_object + .serialize(&mut ser) + .context(error::ManifestCanonicalizeSnafu)?; + + Ok(canonicalized_manifest) + } + + /// Push a single-arch image in oci archive format + pub async fn push_oci_archive(&self, path: &Path, uri: &str) -> Result<()> { + self.image_tool_impl.push_oci_archive(path, uri).await + } + + /// Push the multi-arch kit manifest list + pub async fn push_multi_platform_manifest( + &self, + platform_images: Vec<(DockerArchitecture, String)>, + uri: &str, + ) -> Result<()> { + self.image_tool_impl + .push_multi_platform_manifest(platform_images, uri) + .await + } +} + #[async_trait] -pub trait ImageTool { +pub trait ImageToolImpl: std::fmt::Debug { /// Pull an image archive to disk async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()>; /// Fetch the image config @@ -44,37 +159,6 @@ pub trait ImageTool { ) -> Result<()>; } -/// Auto-select the container tool to use by environment variable -/// and-or auto detection -pub fn image_tool() -> Result> { - if let Ok(name) = env::var("TWOLITER_KIT_IMAGE_TOOL") { - return match name.as_str() { - "docker" => Ok(Rc::new(DockerCLI { - cli: CommandLine { - path: which("docker").context(error::NotFoundSnafu { name: "docker" })?, - }, - })), - tool @ ("crane" | "gcrane" | "krane") => Ok(Rc::new(CraneCLI { - cli: CommandLine { - path: which(tool).context(error::NotFoundSnafu { name: tool })?, - }, - })), - _ => error::UnsupportedSnafu { name }.fail(), - }; - } - let crane = which("krane").or(which("gcrane")).or(which("crane")); - if let Ok(path) = crane { - return Ok(Rc::new(CraneCLI { - cli: CommandLine { path }, - })); - }; - Ok(Rc::new(DockerCLI { - cli: CommandLine { - path: which("docker").context(error::NoneFoundSnafu)?, - }, - })) -} - #[derive(Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "kebab-case")] pub enum DockerArchitecture { @@ -151,6 +235,12 @@ pub mod error { #[snafu(display("invalid architecture '{value}'"))] InvalidArchitecture { value: String }, + #[snafu(display("Failed to deserialize image manifest: {source}"))] + ManifestDeserialize { source: serde_json::Error }, + + #[snafu(display("Failed to canonicalize image manifest: {source}"))] + ManifestCanonicalize { source: serde_json::Error }, + #[snafu(display("No digest returned by `docker load`"))] NoDigest, diff --git a/tools/pubsys/src/kit/publish_kit/mod.rs b/tools/pubsys/src/kit/publish_kit/mod.rs index e1d6062f4..a32a4c774 100644 --- a/tools/pubsys/src/kit/publish_kit/mod.rs +++ b/tools/pubsys/src/kit/publish_kit/mod.rs @@ -1,11 +1,10 @@ use crate::Args; use clap::Parser; use log::{debug, info, trace}; -use oci_cli_wrapper::{image_tool, DockerArchitecture, ImageTool}; +use oci_cli_wrapper::{DockerArchitecture, ImageTool}; use pubsys_config::InfraConfig; use snafu::{ensure, OptionExt, ResultExt}; use std::path::PathBuf; -use std::rc::Rc; /// Takes a local kit built using buildsys and publishes it to a vendor specified in Infra.toml #[derive(Debug, Parser)] @@ -28,20 +27,20 @@ pub(crate) struct PublishKitArgs { } pub(crate) async fn run(args: &Args, publish_kit_args: &PublishKitArgs) -> Result<()> { - let image_tool = image_tool().context(error::ImageToolSnafu)?; + let image_tool = ImageTool::from_environment().context(error::ImageToolSnafu)?; // If a lock file exists, use that, otherwise use Infra.toml let infra_config = InfraConfig::from_path_or_lock(&args.infra_config_path, false) .context(error::ConfigSnafu)?; trace!("Parsed infra config: {:?}", infra_config); - publish_kit(infra_config, publish_kit_args, image_tool).await + publish_kit(infra_config, publish_kit_args, &image_tool).await } async fn publish_kit( infra_config: InfraConfig, publish_kit_args: &PublishKitArgs, - image_tool: Rc, + image_tool: &ImageTool, ) -> Result<()> { // Fetch the vendor container registry uri let vendor = infra_config diff --git a/twoliter/src/lock.rs b/twoliter/src/lock.rs index 6905c07ed..782fea616 100644 --- a/twoliter/src/lock.rs +++ b/twoliter/src/lock.rs @@ -3,7 +3,7 @@ use crate::project::{Image, Project, ValidIdentifier, Vendor}; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use base64::Engine; -use oci_cli_wrapper::{image_tool, DockerArchitecture, ImageTool}; +use oci_cli_wrapper::{DockerArchitecture, ImageTool}; use olpc_cjson::CanonicalFormatter as CanonicalJsonFormatter; use semver::Version; use serde::de::Error; @@ -16,7 +16,6 @@ use std::fs::File; use std::hash::{Hash, Hasher}; use std::mem::take; use std::path::{Path, PathBuf}; -use std::rc::Rc; use tar::Archive as TarArchive; use tokio::fs::read_to_string; @@ -40,13 +39,9 @@ pub(crate) struct LockedImage { } impl LockedImage { - pub async fn new( - image_tool: Rc, - vendor: &Vendor, - image: &Image, - ) -> Result { + pub async fn new(image_tool: &ImageTool, vendor: &Vendor, image: &Image) -> Result { let source = format!("{}/{}:v{}", vendor.registry, image.name, image.version); - let manifest_bytes = image_tool.as_ref().get_manifest(source.as_str()).await?; + let manifest_bytes = image_tool.get_manifest(source.as_str()).await?; // We calculate a 'digest' of the manifest to use as our unique id let digest = sha2::Sha256::digest(manifest_bytes.as_slice()); @@ -188,13 +183,12 @@ impl OCIArchive { self.cache_dir.join(self.digest.replace(':', "-")) } - async fn pull_image(&self, image_tool: Rc) -> Result<()> { + async fn pull_image(&self, image_tool: &ImageTool) -> Result<()> { let digest_uri = self.image.digest_uri(self.digest.as_str()); let oci_archive_path = self.archive_path(); if !oci_archive_path.exists() { create_dir_all(&oci_archive_path).await?; image_tool - .as_ref() .pull_oci_image(oci_archive_path.as_path(), digest_uri.as_str()) .await?; } @@ -312,20 +306,15 @@ impl Lock { /// Fetches all external kits defined in a Twoliter.lock to the build directory pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { - let image_tool = image_tool()?; + let image_tool = ImageTool::from_environment()?; let target_dir = project.external_kits_dir(); create_dir_all(&target_dir).await.context(format!( "failed to create external-kits directory at {}", target_dir.display() ))?; for image in self.kit.iter() { - self.extract_kit( - image_tool.clone(), - &project.external_kits_dir(), - image, - arch, - ) - .await?; + self.extract_kit(&image_tool, &project.external_kits_dir(), image, arch) + .await?; } let mut kit_list = Vec::new(); let mut ser = @@ -357,7 +346,7 @@ impl Lock { async fn get_manifest( &self, - image_tool: Rc, + image_tool: &ImageTool, image: &LockedImage, arch: &str, ) -> Result { @@ -378,7 +367,7 @@ impl Lock { async fn extract_kit

( &self, - image_tool: Rc, + image_tool: &ImageTool, path: P, image: &LockedImage, arch: &str, @@ -394,11 +383,11 @@ impl Lock { create_dir_all(&cache_path).await?; // First get the manifest for the specific requested architecture - let manifest = self.get_manifest(image_tool.clone(), image, arch).await?; + let manifest = self.get_manifest(image_tool, image, arch).await?; let oci_archive = OCIArchive::new(image, manifest.digest.as_str(), &cache_path)?; // Checks for the saved image locally, or else pulls and saves it - oci_archive.pull_image(image_tool.clone()).await?; + oci_archive.pull_image(image_tool).await?; // Checks if this archive has already been extracted by checking a digest file // otherwise cleans up the path and unpacks the archive @@ -411,7 +400,7 @@ impl Lock { let vendor_table = project.vendor(); let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); - let image_tool = image_tool()?; + let image_tool = ImageTool::from_environment()?; let mut remaining: Vec = project.kits(); let mut sdk_set: HashSet = HashSet::new(); @@ -440,8 +429,8 @@ impl Lock { (image.name.clone(), image.vendor.clone()), image.version.clone(), ); - let locked_image = LockedImage::new(image_tool.clone(), vendor, image).await?; - let kit = Self::find_kit(image_tool.clone(), vendor, &locked_image).await?; + let locked_image = LockedImage::new(&image_tool, vendor, image).await?; + let kit = Self::find_kit(&image_tool, vendor, &locked_image).await?; locked.push(locked_image); sdk_set.insert(kit.sdk); for dep in kit.kits { @@ -470,13 +459,13 @@ impl Lock { schema_version: project.schema_version(), release_version: project.release_version().to_string(), digest: project.digest()?, - sdk: LockedImage::new(image_tool, vendor, sdk).await?, + sdk: LockedImage::new(&image_tool, vendor, sdk).await?, kit: locked, }) } async fn find_kit( - image_tool: Rc, + image_tool: &ImageTool, vendor: &Vendor, image: &LockedImage, ) -> Result { @@ -487,7 +476,7 @@ impl Lock { let image_uri = format!("{}/{}@{}", vendor.registry, image.name, manifest.digest); // Now we want to fetch the metadata from the OCI image config - let config = image_tool.as_ref().get_config(image_uri.as_str()).await?; + let config = image_tool.get_config(image_uri.as_str()).await?; let encoded = config .labels .get("dev.bottlerocket.kit.v1") From 6f0ebf6dc8e2645b8f8bdf7b62cd9600c5760f6d Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Wed, 17 Jul 2024 00:19:26 +0000 Subject: [PATCH 3/3] add integration test for `twoliter update` --- .github/actions/install-crane/action.yaml | 45 +++++++++++ .github/workflows/release.yml | 4 + .github/workflows/rust.yml | 4 + Cargo.lock | 8 ++ Cargo.toml | 2 + Makefile | 6 +- tests/integration-tests/Cargo.toml | 9 +++ tests/integration-tests/src/lib.rs | 44 +++++++++++ .../integration-tests/src/twoliter_update.rs | 76 +++++++++++++++++++ tests/projects/external-kit/Twoliter.toml | 10 +-- 10 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 .github/actions/install-crane/action.yaml create mode 100644 tests/integration-tests/Cargo.toml create mode 100644 tests/integration-tests/src/lib.rs create mode 100644 tests/integration-tests/src/twoliter_update.rs diff --git a/.github/actions/install-crane/action.yaml b/.github/actions/install-crane/action.yaml new file mode 100644 index 000000000..194fcb7d7 --- /dev/null +++ b/.github/actions/install-crane/action.yaml @@ -0,0 +1,45 @@ +name: "Install crane" +description: "Installs crane for use in testing." +inputs: + crane-version: + description: "Version of crane to install" + required: false + default: latest + install-dir: + description: "Directory to install crane" + required: false + default: $HOME/.crane + +runs: + using: "composite" + steps: + - shell: bash + run: | + mkdir -p ${{ inputs.install-dir }} + + VERSION=${{ inputs.crane-version }} + if [[ "${VERSION}" == "latest" ]]; then + VERSION=$(gh release list \ + --exclude-pre-releases \ + -R google/go-containerregistry \ + --json name \ + | jq -r '.[0].name') + fi + + case ${{ runner.arch }} in + X64) + ARCH=x86_64 + ;; + ARM64) + ARCH=arm64 + ;; + esac + + ARTIFACT_NAME="go-containerregistry_Linux_${ARCH}.tar.gz" + gh release download "${VERSION}" \ + -R google/go-containerregistry \ + -p "${ARTIFACT_NAME}" \ + --output - \ + | tar -zxvf - -C "${{ inputs.install-dir }}" crane + + echo "${{ inputs.install-dir }}" >> "${GITHUB_PATH}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7f0de063c..8f853bdcf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive + - name: Install crane for testing + uses: ./.github/actions/install-crane + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install cargo-dist run: | cargo install --locked \ diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e784ba945..28cd6e062 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -22,6 +22,10 @@ jobs: labels: bottlerocket_ubuntu-latest_16-core steps: - uses: actions/checkout@v3 + - name: Install crane for testing + uses: ./.github/actions/install-crane + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: cargo install cargo-deny --locked - run: cargo install cargo-make --locked - run: make build diff --git a/Cargo.lock b/Cargo.lock index 17dc5cc8c..48e551449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1833,6 +1833,14 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integration-tests" +version = "0.1.0" +dependencies = [ + "tokio", + "twoliter", +] + [[package]] name = "ipnet" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index 30bc80d18..a0a22017f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,8 @@ members = [ "tools/unplug", "tools/update-metadata", "twoliter", + + "tests/integration-tests", ] [profile.release] diff --git a/Makefile b/Makefile index d078a6521..b1f5c5c35 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,12 @@ fmt: test: cargo test --release --locked +.PHONY: integ +integ: + cargo test --manifest-path tests/integration-tests/Cargo.toml -- --include-ignored + .PHONY: check -check: fmt clippy deny test +check: fmt clippy deny test integ .PHONY: build build: check diff --git a/tests/integration-tests/Cargo.toml b/tests/integration-tests/Cargo.toml new file mode 100644 index 000000000..869c25288 --- /dev/null +++ b/tests/integration-tests/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "integration-tests" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dev-dependencies] +tokio = { version = "1", default-features = false, features = ["process", "fs", "rt-multi-thread"] } +twoliter = { version = "0", path = "../../twoliter", artifact = [ "bin:twoliter" ] } diff --git a/tests/integration-tests/src/lib.rs b/tests/integration-tests/src/lib.rs new file mode 100644 index 000000000..379a7b0f2 --- /dev/null +++ b/tests/integration-tests/src/lib.rs @@ -0,0 +1,44 @@ +#![cfg(test)] + +use std::ffi::OsStr; +use std::path::PathBuf; +use tokio::process::Command; + +mod twoliter_update; + +pub const TWOLITER_PATH: &'static str = env!("CARGO_BIN_FILE_TWOLITER"); + +pub fn test_projects_dir() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.pop(); + p.join("projects") +} + +pub async fn run_command(cmd: S, args: I, env: E) -> std::process::Output +where + I: IntoIterator, + E: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().collect(); + + println!( + "Executing '{}' with args [{}]", + cmd.as_ref().to_string_lossy(), + args.iter() + .map(|arg| format!("'{}'", arg.as_ref().to_string_lossy())) + .collect::>() + .join(", ") + ); + + let output = Command::new(cmd) + .args(args.into_iter()) + .envs(env.into_iter()) + .output() + .await + .expect("failed to execute process"); + + println!("stdout: {}", String::from_utf8_lossy(&output.stdout)); + println!("stderr: {}", String::from_utf8_lossy(&output.stderr)); + output +} diff --git a/tests/integration-tests/src/twoliter_update.rs b/tests/integration-tests/src/twoliter_update.rs new file mode 100644 index 000000000..a565aa5cf --- /dev/null +++ b/tests/integration-tests/src/twoliter_update.rs @@ -0,0 +1,76 @@ +use super::{run_command, test_projects_dir, TWOLITER_PATH}; + +const EXPECTED_LOCKFILE: &str = r#"schema-version = 1 +release-version = "1.0.0" +digest = "m/6DbBacnIBHMo34GCuzA4pAHzrnQJ2G/XJMMguZXjw=" + +[sdk] +name = "bottlerocket-sdk" +version = "0.42.0" +vendor = "bottlerocket" +source = "public.ecr.aws/bottlerocket/bottlerocket-sdk:v0.42.0" +digest = "myHHKE41h9qfeyR6V6HB0BfiLPwj3QEFLUFy4TXcR10=" + +[[kit]] +name = "bottlerocket-core-kit" +version = "2.0.0" +vendor = "custom-vendor" +source = "public.ecr.aws/bottlerocket/bottlerocket-core-kit:v2.0.0" +digest = "vlTsAAbSCzXFZofVmw8pLLkRjnG/y8mtb2QsQBSz1zk=" +"#; + +#[tokio::test] +#[ignore] +/// Generates a Twoliter.lock file for the `external-kit` project using docker +async fn test_twoliter_update_docker() { + let external_kit = test_projects_dir().join("external-kit"); + + let lockfile = external_kit.join("Twoliter.lock"); + tokio::fs::remove_file(&lockfile).await.ok(); + + let output = run_command( + TWOLITER_PATH, + [ + "update", + "--project-path", + external_kit.join("Twoliter.toml").to_str().unwrap(), + ], + [("TWOLITER_KIT_IMAGE_TOOL", "docker")], + ) + .await; + + assert!(output.status.success()); + + let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap(); + assert_eq!(lock_contents, EXPECTED_LOCKFILE); + + tokio::fs::remove_file(&lockfile).await.ok(); +} + +#[tokio::test] +#[ignore] +/// Generates a Twoliter.lock file for the `external-kit` project using crane +async fn test_twoliter_update_crane() { + let external_kit = test_projects_dir().join("external-kit"); + + let lockfile = external_kit.join("Twoliter.lock"); + tokio::fs::remove_file(&lockfile).await.ok(); + + let output = run_command( + TWOLITER_PATH, + [ + "update", + "--project-path", + external_kit.join("Twoliter.toml").to_str().unwrap(), + ], + [("TWOLITER_KIT_IMAGE_TOOL", "crane")], + ) + .await; + + assert!(output.status.success()); + + let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap(); + assert_eq!(lock_contents, EXPECTED_LOCKFILE); + + tokio::fs::remove_file(&lockfile).await.ok(); +} diff --git a/tests/projects/external-kit/Twoliter.toml b/tests/projects/external-kit/Twoliter.toml index 851923590..ea47adb8d 100644 --- a/tests/projects/external-kit/Twoliter.toml +++ b/tests/projects/external-kit/Twoliter.toml @@ -1,19 +1,13 @@ schema-version = 1 release-version = "1.0.0" -[sdk] -name = "bottlerocket-sdk" -vendor = "bottlerocket" -version = "0.41.0" - [vendor.bottlerocket] registry = "public.ecr.aws/bottlerocket" [vendor.custom-vendor] -# We need to figure out how we can do integration testing with this registry = "public.ecr.aws/bottlerocket" [[kit]] -name = "core-kit" -version = "0.1.0" +name = "bottlerocket-core-kit" +version = "2.0.0" vendor = "custom-vendor"