Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix image handling bugs in twoliter update #326

Merged
merged 3 commits into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/actions/install-crane/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: "Install crane"
description: "Installs crane for use in testing."
inputs:
crane-version:
description: "Version of crane to install"
required: false
default: latest
install-dir:
description: "Directory to install crane"
required: false
default: $HOME/.crane

runs:
using: "composite"
steps:
- shell: bash
run: |
mkdir -p ${{ inputs.install-dir }}

VERSION=${{ inputs.crane-version }}
if [[ "${VERSION}" == "latest" ]]; then
VERSION=$(gh release list \
--exclude-pre-releases \
-R google/go-containerregistry \
--json name \
| jq -r '.[0].name')
fi

case ${{ runner.arch }} in
X64)
ARCH=x86_64
;;
ARM64)
ARCH=arm64
;;
esac

ARTIFACT_NAME="go-containerregistry_Linux_${ARCH}.tar.gz"
gh release download "${VERSION}" \
-R google/go-containerregistry \
-p "${ARTIFACT_NAME}" \
--output - \
| tar -zxvf - -C "${{ inputs.install-dir }}" crane

echo "${{ inputs.install-dir }}" >> "${GITHUB_PATH}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice!

4 changes: 4 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ jobs:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Install crane for testing
uses: ./.github/actions/install-crane
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Install cargo-dist
run: |
cargo install --locked \
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ jobs:
labels: bottlerocket_ubuntu-latest_16-core
steps:
- uses: actions/checkout@v3
- name: Install crane for testing
uses: ./.github/actions/install-crane
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: cargo install cargo-deny --locked
- run: cargo install cargo-make --locked
- run: make build
10 changes: 10 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ members = [
"tools/unplug",
"tools/update-metadata",
"twoliter",

"tests/integration-tests",
]

[profile.release]
Expand Down
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,12 @@ fmt:
test:
cargo test --release --locked

.PHONY: integ
integ:
cargo test --manifest-path tests/integration-tests/Cargo.toml -- --include-ignored

.PHONY: check
check: fmt clippy deny test
check: fmt clippy deny test integ

.PHONY: build
build: check
Expand Down
9 changes: 9 additions & 0 deletions tests/integration-tests/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[package]
name = "integration-tests"
version = "0.1.0"
edition = "2021"
license = "MIT OR Apache-2.0"

[dev-dependencies]
tokio = { version = "1", default-features = false, features = ["process", "fs", "rt-multi-thread"] }
twoliter = { version = "0", path = "../../twoliter", artifact = [ "bin:twoliter" ] }
44 changes: 44 additions & 0 deletions tests/integration-tests/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#![cfg(test)]

use std::ffi::OsStr;
use std::path::PathBuf;
use tokio::process::Command;

mod twoliter_update;

pub const TWOLITER_PATH: &'static str = env!("CARGO_BIN_FILE_TWOLITER");

pub fn test_projects_dir() -> PathBuf {
let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
p.pop();
p.join("projects")
}

pub async fn run_command<I, S, E>(cmd: S, args: I, env: E) -> std::process::Output
where
I: IntoIterator<Item = S>,
E: IntoIterator<Item = (S, S)>,
S: AsRef<OsStr>,
{
let args: Vec<S> = args.into_iter().collect();

println!(
"Executing '{}' with args [{}]",
cmd.as_ref().to_string_lossy(),
args.iter()
.map(|arg| format!("'{}'", arg.as_ref().to_string_lossy()))
.collect::<Vec<_>>()
.join(", ")
);

let output = Command::new(cmd)
.args(args.into_iter())
.envs(env.into_iter())
.output()
.await
.expect("failed to execute process");

println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
println!("stderr: {}", String::from_utf8_lossy(&output.stderr));
output
}
76 changes: 76 additions & 0 deletions tests/integration-tests/src/twoliter_update.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
use super::{run_command, test_projects_dir, TWOLITER_PATH};

const EXPECTED_LOCKFILE: &str = r#"schema-version = 1
release-version = "1.0.0"
digest = "m/6DbBacnIBHMo34GCuzA4pAHzrnQJ2G/XJMMguZXjw="

[sdk]
name = "bottlerocket-sdk"
version = "0.42.0"
vendor = "bottlerocket"
source = "public.ecr.aws/bottlerocket/bottlerocket-sdk:v0.42.0"
digest = "myHHKE41h9qfeyR6V6HB0BfiLPwj3QEFLUFy4TXcR10="

[[kit]]
name = "bottlerocket-core-kit"
version = "2.0.0"
vendor = "custom-vendor"
source = "public.ecr.aws/bottlerocket/bottlerocket-core-kit:v2.0.0"
digest = "vlTsAAbSCzXFZofVmw8pLLkRjnG/y8mtb2QsQBSz1zk="
"#;

#[tokio::test]
#[ignore]
/// Generates a Twoliter.lock file for the `external-kit` project using docker
async fn test_twoliter_update_docker() {
let external_kit = test_projects_dir().join("external-kit");

let lockfile = external_kit.join("Twoliter.lock");
tokio::fs::remove_file(&lockfile).await.ok();

let output = run_command(
TWOLITER_PATH,
[
"update",
"--project-path",
external_kit.join("Twoliter.toml").to_str().unwrap(),
],
[("TWOLITER_KIT_IMAGE_TOOL", "docker")],
)
.await;

assert!(output.status.success());

let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
assert_eq!(lock_contents, EXPECTED_LOCKFILE);

tokio::fs::remove_file(&lockfile).await.ok();
}

#[tokio::test]
#[ignore]
/// Generates a Twoliter.lock file for the `external-kit` project using crane
async fn test_twoliter_update_crane() {
let external_kit = test_projects_dir().join("external-kit");

let lockfile = external_kit.join("Twoliter.lock");
tokio::fs::remove_file(&lockfile).await.ok();

let output = run_command(
TWOLITER_PATH,
[
"update",
"--project-path",
external_kit.join("Twoliter.toml").to_str().unwrap(),
],
[("TWOLITER_KIT_IMAGE_TOOL", "crane")],
)
.await;

assert!(output.status.success());

let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
assert_eq!(lock_contents, EXPECTED_LOCKFILE);

tokio::fs::remove_file(&lockfile).await.ok();
}
10 changes: 2 additions & 8 deletions tests/projects/external-kit/Twoliter.toml
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
schema-version = 1
release-version = "1.0.0"

[sdk]
name = "bottlerocket-sdk"
vendor = "bottlerocket"
version = "0.41.0"

[vendor.bottlerocket]
registry = "public.ecr.aws/bottlerocket"

[vendor.custom-vendor]
# We need to figure out how we can do integration testing with this
registry = "public.ecr.aws/bottlerocket"

[[kit]]
name = "core-kit"
version = "0.1.0"
name = "bottlerocket-core-kit"
version = "2.0.0"
vendor = "custom-vendor"
2 changes: 2 additions & 0 deletions tools/oci-cli-wrapper/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ publish = false

[dependencies]
async-trait = "0.1"
log = "0.4"
olpc-cjson = "0.1"
regex = "1"
serde = { version = "1", features = ["derive"]}
serde_json = "1"
Expand Down
21 changes: 21 additions & 0 deletions tools/oci-cli-wrapper/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,21 @@ use tokio::process::Command;

use crate::{error, Result};

#[derive(Debug)]
pub(crate) struct CommandLine {
pub(crate) path: PathBuf,
}

impl CommandLine {
pub(crate) async fn output(&self, args: &[&str], error_msg: String) -> Result<Vec<u8>> {
log::debug!(
"Executing '{}' with args [{}]",
self.path.display(),
args.iter()
.map(|arg| format!("'{}'", arg))
.collect::<Vec<_>>()
.join(", ")
);
let output = Command::new(&self.path)
.args(args)
.output()
Expand All @@ -23,10 +32,22 @@ impl CommandLine {
args: args.iter().map(|x| x.to_string()).collect::<Vec<_>>()
}
);
log::debug!(
"stdout: {}",
String::from_utf8_lossy(&output.stdout).to_string()
);
Ok(output.stdout)
}

pub(crate) async fn spawn(&self, args: &[&str], error_msg: String) -> Result<()> {
log::debug!(
"Executing '{}' with args [{}]",
self.path.display(),
args.iter()
.map(|arg| format!("'{}'", arg))
.collect::<Vec<_>>()
.join(", ")
);
let status = Command::new(&self.path)
.args(args)
.spawn()
Expand Down
5 changes: 3 additions & 2 deletions tools/oci-cli-wrapper/src/crane.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,16 @@ use tar::Archive as TarArchive;
use tempfile::TempDir;

use crate::{
cli::CommandLine, error, ConfigView, DockerArchitecture, ImageTool, ImageView, Result,
cli::CommandLine, error, ConfigView, DockerArchitecture, ImageToolImpl, ImageView, Result,
};

#[derive(Debug)]
pub struct CraneCLI {
pub(crate) cli: CommandLine,
}

#[async_trait]
impl ImageTool for CraneCLI {
impl ImageToolImpl for CraneCLI {
async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> {
let archive_path = path.to_string_lossy();
self.cli
Expand Down
13 changes: 4 additions & 9 deletions tools/oci-cli-wrapper/src/docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ use tar::Archive;
use tempfile::NamedTempFile;

use crate::cli::CommandLine;
use crate::{error, ConfigView, DockerArchitecture, ImageTool, Result};
use crate::{error, ConfigView, DockerArchitecture, ImageToolImpl, Result};

#[derive(Debug)]
pub struct DockerCLI {
pub(crate) cli: CommandLine,
}

#[async_trait]
impl ImageTool for DockerCLI {
impl ImageToolImpl for DockerCLI {
async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> {
// First we pull the image to local daemon
self.cli
Expand Down Expand Up @@ -57,13 +58,7 @@ impl ImageTool for DockerCLI {
let bytes = self
.cli
.output(
&[
"image",
"inspect",
uri,
"--format",
"\"{{ json .Config }}\"",
],
&["image", "inspect", uri, "--format", "{{ json .Config }}"],
format!("failed to fetch image config from {}", uri),
)
.await?;
Expand Down
Loading
Loading