Fix image handling bugs in twoliter update

.github/actions/install-crane/action.yaml

@@ -0,0 +1,45 @@
+name: "Install crane"
+description: "Installs crane for use in testing."
+inputs:
+  crane-version:
+    description: "Version of crane to install"
+    required: false
+    default: latest
+  install-dir:
+    description: "Directory to install crane"
+    required: false
+    default: $HOME/.crane
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        mkdir -p ${{ inputs.install-dir }}
+
+        VERSION=${{ inputs.crane-version }}
+        if [[ "${VERSION}" == "latest" ]]; then
+          VERSION=$(gh release list \
+            --exclude-pre-releases \
+            -R google/go-containerregistry \
+            --json name \
+            | jq -r '.[0].name')
+        fi
+
+        case ${{ runner.arch }} in
+          X64)
+            ARCH=x86_64
+            ;;
+          ARM64)
+            ARCH=arm64
+            ;;
+        esac
+
+        ARTIFACT_NAME="go-containerregistry_Linux_${ARCH}.tar.gz"
+        gh release download "${VERSION}" \
+          -R google/go-containerregistry \
+          -p "${ARTIFACT_NAME}" \
+          --output - \
+          | tar -zxvf - -C "${{ inputs.install-dir }}" crane
+
+        echo "${{ inputs.install-dir }}" >> "${GITHUB_PATH}"

.github/workflows/release.yml

@@ -58,6 +58,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
+      - name: Install crane for testing
+        uses: ./.github/actions/install-crane
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Install cargo-dist
         run: |
           cargo install --locked \

.github/workflows/rust.yml

@@ -22,6 +22,10 @@ jobs:
       labels: bottlerocket_ubuntu-latest_16-core
     steps:
       - uses: actions/checkout@v3
+      - name: Install crane for testing
+        uses: ./.github/actions/install-crane
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: cargo install cargo-deny --locked
       - run: cargo install cargo-make --locked
       - run: make build

Cargo.toml

@@ -16,6 +16,8 @@ members = [
     "tools/unplug",
     "tools/update-metadata",
     "twoliter",
+
+    "tests/integration-tests",
 ]
 
 [profile.release]

Makefile

@@ -21,8 +21,12 @@ fmt:
 test:
 	cargo test --release --locked
 
+.PHONY: integ
+integ:
+	cargo test --manifest-path tests/integration-tests/Cargo.toml -- --include-ignored
+
 .PHONY: check
-check: fmt clippy deny test
+check: fmt clippy deny test integ
 
 .PHONY: build
 build: check

tests/integration-tests/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "integration-tests"
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+
+[dev-dependencies]
+tokio = { version = "1", default-features = false, features = ["process", "fs", "rt-multi-thread"] }
+twoliter = { version = "0", path = "../../twoliter", artifact = [ "bin:twoliter" ] }

tests/integration-tests/src/lib.rs

@@ -0,0 +1,44 @@
+#![cfg(test)]
+
+use std::ffi::OsStr;
+use std::path::PathBuf;
+use tokio::process::Command;
+
+mod twoliter_update;
+
+pub const TWOLITER_PATH: &'static str = env!("CARGO_BIN_FILE_TWOLITER");
+
+pub fn test_projects_dir() -> PathBuf {
+    let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    p.pop();
+    p.join("projects")
+}
+
+pub async fn run_command<I, S, E>(cmd: S, args: I, env: E) -> std::process::Output
+where
+    I: IntoIterator<Item = S>,
+    E: IntoIterator<Item = (S, S)>,
+    S: AsRef<OsStr>,
+{
+    let args: Vec<S> = args.into_iter().collect();
+
+    println!(
+        "Executing '{}' with args [{}]",
+        cmd.as_ref().to_string_lossy(),
+        args.iter()
+            .map(|arg| format!("'{}'", arg.as_ref().to_string_lossy()))
+            .collect::<Vec<_>>()
+            .join(", ")
+    );
+
+    let output = Command::new(cmd)
+        .args(args.into_iter())
+        .envs(env.into_iter())
+        .output()
+        .await
+        .expect("failed to execute process");
+
+    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
+    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));
+    output
+}

tests/integration-tests/src/twoliter_update.rs

@@ -0,0 +1,76 @@
+use super::{run_command, test_projects_dir, TWOLITER_PATH};
+
+const EXPECTED_LOCKFILE: &str = r#"schema-version = 1
+release-version = "1.0.0"
+digest = "m/6DbBacnIBHMo34GCuzA4pAHzrnQJ2G/XJMMguZXjw="
+
+[sdk]
+name = "bottlerocket-sdk"
+version = "0.42.0"
+vendor = "bottlerocket"
+source = "public.ecr.aws/bottlerocket/bottlerocket-sdk:v0.42.0"
+digest = "myHHKE41h9qfeyR6V6HB0BfiLPwj3QEFLUFy4TXcR10="
+
+[[kit]]
+name = "bottlerocket-core-kit"
+version = "2.0.0"
+vendor = "custom-vendor"
+source = "public.ecr.aws/bottlerocket/bottlerocket-core-kit:v2.0.0"
+digest = "vlTsAAbSCzXFZofVmw8pLLkRjnG/y8mtb2QsQBSz1zk="
+"#;
+
+#[tokio::test]
+#[ignore]
+/// Generates a Twoliter.lock file for the `external-kit` project using docker
+async fn test_twoliter_update_docker() {
+    let external_kit = test_projects_dir().join("external-kit");
+
+    let lockfile = external_kit.join("Twoliter.lock");
+    tokio::fs::remove_file(&lockfile).await.ok();
+
+    let output = run_command(
+        TWOLITER_PATH,
+        [
+            "update",
+            "--project-path",
+            external_kit.join("Twoliter.toml").to_str().unwrap(),
+        ],
+        [("TWOLITER_KIT_IMAGE_TOOL", "docker")],
+    )
+    .await;
+
+    assert!(output.status.success());
+
+    let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
+    assert_eq!(lock_contents, EXPECTED_LOCKFILE);
+
+    tokio::fs::remove_file(&lockfile).await.ok();
+}
+
+#[tokio::test]
+#[ignore]
+/// Generates a Twoliter.lock file for the `external-kit` project using crane
+async fn test_twoliter_update_crane() {
+    let external_kit = test_projects_dir().join("external-kit");
+
+    let lockfile = external_kit.join("Twoliter.lock");
+    tokio::fs::remove_file(&lockfile).await.ok();
+
+    let output = run_command(
+        TWOLITER_PATH,
+        [
+            "update",
+            "--project-path",
+            external_kit.join("Twoliter.toml").to_str().unwrap(),
+        ],
+        [("TWOLITER_KIT_IMAGE_TOOL", "crane")],
+    )
+    .await;
+
+    assert!(output.status.success());
+
+    let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
+    assert_eq!(lock_contents, EXPECTED_LOCKFILE);
+
+    tokio::fs::remove_file(&lockfile).await.ok();
+}

tests/projects/external-kit/Twoliter.toml

@@ -1,19 +1,13 @@
 schema-version = 1
 release-version = "1.0.0"
 
-[sdk]
-name = "bottlerocket-sdk"
-vendor = "bottlerocket"
-version = "0.41.0"
-
 [vendor.bottlerocket]
 registry = "public.ecr.aws/bottlerocket"
 
 [vendor.custom-vendor]
-# We need to figure out how we can do integration testing with this
 registry = "public.ecr.aws/bottlerocket"
 
 [[kit]]
-name = "core-kit"
-version = "0.1.0"
+name = "bottlerocket-core-kit"
+version = "2.0.0"
 vendor = "custom-vendor"

tools/oci-cli-wrapper/Cargo.toml

@@ -8,6 +8,8 @@ publish = false
 
 [dependencies]
 async-trait = "0.1"
+log = "0.4"
+olpc-cjson = "0.1"
 regex = "1"
 serde = { version = "1", features = ["derive"]}
 serde_json = "1"

tools/oci-cli-wrapper/src/cli.rs

@@ -4,12 +4,21 @@ use tokio::process::Command;
 
 use crate::{error, Result};
 
+#[derive(Debug)]
 pub(crate) struct CommandLine {
     pub(crate) path: PathBuf,
 }
 
 impl CommandLine {
     pub(crate) async fn output(&self, args: &[&str], error_msg: String) -> Result<Vec<u8>> {
+        log::debug!(
+            "Executing '{}' with args [{}]",
+            self.path.display(),
+            args.iter()
+                .map(|arg| format!("'{}'", arg))
+                .collect::<Vec<_>>()
+                .join(", ")
+        );
         let output = Command::new(&self.path)
             .args(args)
             .output()
@@ -23,10 +32,22 @@ impl CommandLine {
                 args: args.iter().map(|x| x.to_string()).collect::<Vec<_>>()
             }
         );
+        log::debug!(
+            "stdout: {}",
+            String::from_utf8_lossy(&output.stdout).to_string()
+        );
         Ok(output.stdout)
     }
 
     pub(crate) async fn spawn(&self, args: &[&str], error_msg: String) -> Result<()> {
+        log::debug!(
+            "Executing '{}' with args [{}]",
+            self.path.display(),
+            args.iter()
+                .map(|arg| format!("'{}'", arg))
+                .collect::<Vec<_>>()
+                .join(", ")
+        );
         let status = Command::new(&self.path)
             .args(args)
             .spawn()

tools/oci-cli-wrapper/src/crane.rs

@@ -7,15 +7,16 @@ use tar::Archive as TarArchive;
 use tempfile::TempDir;
 
 use crate::{
-    cli::CommandLine, error, ConfigView, DockerArchitecture, ImageTool, ImageView, Result,
+    cli::CommandLine, error, ConfigView, DockerArchitecture, ImageToolImpl, ImageView, Result,
 };
 
+#[derive(Debug)]
 pub struct CraneCLI {
     pub(crate) cli: CommandLine,
 }
 
 #[async_trait]
-impl ImageTool for CraneCLI {
+impl ImageToolImpl for CraneCLI {
     async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> {
         let archive_path = path.to_string_lossy();
         self.cli

tools/oci-cli-wrapper/src/docker.rs

@@ -8,14 +8,15 @@ use tar::Archive;
 use tempfile::NamedTempFile;
 
 use crate::cli::CommandLine;
-use crate::{error, ConfigView, DockerArchitecture, ImageTool, Result};
+use crate::{error, ConfigView, DockerArchitecture, ImageToolImpl, Result};
 
+#[derive(Debug)]
 pub struct DockerCLI {
     pub(crate) cli: CommandLine,
 }
 
 #[async_trait]
-impl ImageTool for DockerCLI {
+impl ImageToolImpl for DockerCLI {
     async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> {
         // First we pull the image to local daemon
         self.cli
@@ -57,13 +58,7 @@ impl ImageTool for DockerCLI {
         let bytes = self
             .cli
             .output(
-                &[
-                    "image",
-                    "inspect",
-                    uri,
-                    "--format",
-                    "\"{{ json .Config }}\"",
-                ],
+                &["image", "inspect", uri, "--format", "{{ json .Config }}"],
                 format!("failed to fetch image config from {}", uri),
             )
             .await?;