From 48616c7f45151e0577a994f69b47000fe32c0410 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Jun 2024 14:29:22 -0400 Subject: [PATCH 1/3] cli: Add a new `bootc image` subcommand We have a basic `bootc image list` but more interesting is `bootc image push` which defaults to copying the booted image into the container storage. Signed-off-by: Colin Walters --- lib/Cargo.toml | 2 +- lib/src/cli.rs | 71 +++++++++++++++++++ lib/src/image.rs | 68 ++++++++++++++++++ lib/src/lib.rs | 1 + .../booted/002-test-image-pushpull-upgrade.nu | 56 +++++++++++++++ 5 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 lib/src/image.rs create mode 100644 tests/booted/002-test-image-pushpull-upgrade.nu diff --git a/lib/Cargo.toml b/lib/Cargo.toml index dd5ed196..6ae4633d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -16,7 +16,7 @@ anstream = "0.6.13" anstyle = "1.0.6" anyhow = "1.0.82" camino = { version = "1.1.6", features = ["serde1"] } -ostree-ext = { version = "0.14.0" } +ostree-ext = { version = "0.14.0" } chrono = { version = "0.4.38", features = ["serde"] } clap = { version= "4.5.4", features = ["derive","cargo"] } clap_mangen = { version = "0.2.20", optional = true } diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 7346b247..e193049b 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -17,6 +17,7 @@ use fn_error_context::context; use ostree::gio; use ostree_container::store::PrepareResult; use ostree_ext::container as ostree_container; +use ostree_ext::container::Transport; use ostree_ext::keyfileext::KeyFileExt; use ostree_ext::ostree; @@ -191,6 +192,59 @@ pub(crate) enum ContainerOpts { Lint, } +/// Subcommands which operate on images. +#[derive(Debug, clap::Subcommand, PartialEq, Eq)] +pub(crate) enum ImageOpts { + /// List fetched images stored in the bootc storage. + /// + /// Note that these are distinct from images stored via e.g. `podman`. + List, + /// Copy a container image from the bootc storage to a target. + /// + /// ## Copying the booted container image to containers-storage: (podman) + /// + /// The source and target are both optional; if both are left unspecified, + /// via a simple invocation of `bootc image push`, then the default is to + /// push the currently booted image to `containers-storage` (as used by podman, etc.) + /// and tagged with the image name `localhost/bootc`, + /// + /// ## Copying the booted container image to a remote registry + /// + /// Aside from the special case above, default transport is `registry`. This + /// means that an invocation of + /// + /// `bootc image push --target quay.io/example/someimage:latest` will push the + /// booted container image to the target registry. This will be done via an + /// invocation equivalent to `skopeo copy`, and hence the defaults for that apply. + /// For example, the default registry authentication file applies. + /// + /// ## Copying a non-default container image + /// + /// It is also possible to copy an image other than the currently booted one by + /// specifying `--source`. + /// + /// ## Pulling images + /// + /// At the current time there is no explicit support for pulling images other than indirectly + /// via e.g. `bootc switch` or `bootc upgrade`. + Push { + /// The transport; e.g. oci, oci-archive, containers-storage. Defaults to `registry`. + /// + /// For more information, see `man containers-transports`. + #[clap(long, default_value = "registry")] + transport: String, + + #[clap(long)] + /// The source image; if not specified, the booted image will be used. + source: Option, + + #[clap(long)] + /// The destination; if not specified, then the default is to push to `containers-storage:localhost/bootc`; + /// this will make the image accessible via e.g. `podman run localhost/bootc` and for builds. + target: Option, + }, +} + /// Hidden, internal only options #[derive(Debug, clap::Subcommand, PartialEq, Eq)] pub(crate) enum InternalsOpts { @@ -321,6 +375,12 @@ pub(crate) enum Opt { /// Operations which can be executed as part of a container build. #[clap(subcommand)] Container(ContainerOpts), + /// Operations on container images + /// + /// Stability: This interface is not declared stable and may change or be removed + /// at any point in the future. + #[clap(subcommand, hide = true)] + Image(ImageOpts), /// Execute the given command in the host mount namespace #[cfg(feature = "install")] #[clap(hide = true)] @@ -732,6 +792,17 @@ async fn run_from_opt(opt: Opt) -> Result<()> { Ok(()) } }, + Opt::Image(opts) => match opts { + ImageOpts::List => crate::image::list_entrypoint().await, + ImageOpts::Push { + transport, + source, + target, + } => { + let transport = Transport::try_from(transport.as_str())?; + crate::image::push_entrypoint(transport, source.as_deref(), target.as_deref()).await + } + }, #[cfg(feature = "install")] Opt::Install(opts) => match opts { InstallOpts::ToDisk(opts) => crate::install::install_to_disk(opts).await, diff --git a/lib/src/image.rs b/lib/src/image.rs new file mode 100644 index 00000000..d79ed605 --- /dev/null +++ b/lib/src/image.rs @@ -0,0 +1,68 @@ +//! # Controlling bootc-managed images +//! +//! APIs for operating on container images in the bootc storage. + +use anyhow::{Context, Result}; +use fn_error_context::context; +use ostree_ext::container::{ImageReference, Transport}; + +/// The name of the image we push to containers-storage if nothing is specified. +const IMAGE_DEFAULT: &str = "localhost/bootc"; + +#[context("Listing images")] +pub(crate) async fn list_entrypoint() -> Result<()> { + let sysroot = crate::cli::get_locked_sysroot().await?; + let repo = &sysroot.repo(); + + let images = ostree_ext::container::store::list_images(repo).context("Querying images")?; + + for image in images { + println!("{image}"); + } + Ok(()) +} + +#[context("Pushing image")] +pub(crate) async fn push_entrypoint( + transport: Transport, + source: Option<&str>, + target: Option<&str>, +) -> Result<()> { + let sysroot = crate::cli::get_locked_sysroot().await?; + + let repo = &sysroot.repo(); + + // If the target isn't specified, push to containers-storage + our default image + let target = if let Some(target) = target { + ImageReference { + transport, + name: target.to_owned(), + } + } else { + ImageReference { + transport: Transport::ContainerStorage, + name: IMAGE_DEFAULT.to_string(), + } + }; + + // If the source isn't specified, we use the booted image + let source = if let Some(source) = source { + ImageReference::try_from(source).context("Parsing source image")? + } else { + let status = crate::status::get_status_require_booted(&sysroot)?; + // SAFETY: We know it's booted + let booted = status.2.status.booted.unwrap(); + let booted_image = booted.image.unwrap().image; + ImageReference { + transport: Transport::try_from(booted_image.transport.as_str()).unwrap(), + name: booted_image.image, + } + }; + let mut opts = ostree_ext::container::store::ExportToOCIOpts::default(); + opts.progress_to_stdout = true; + println!("Copying local image {source} to {target} ..."); + let r = ostree_ext::container::store::export(repo, &source, &target, Some(opts)).await?; + + println!("Pushed: {target} {r}"); + Ok(()) +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index f2f2c60d..9f8d4ac5 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -20,6 +20,7 @@ pub mod cli; pub(crate) mod deploy; pub(crate) mod generator; +mod image; pub(crate) mod journal; pub(crate) mod kargs; mod lints; diff --git a/tests/booted/002-test-image-pushpull-upgrade.nu b/tests/booted/002-test-image-pushpull-upgrade.nu new file mode 100644 index 00000000..3d9e3bbf --- /dev/null +++ b/tests/booted/002-test-image-pushpull-upgrade.nu @@ -0,0 +1,56 @@ +# This test does: +# bootc image push +# podman build +# bootc switch +use std assert +use tap.nu + +# This code runs on *each* boot. +# Here we just capture information. +bootc status +let st = bootc status --json | from json +let booted = $st.status.booted.image.image + +# Run on the first boot +def initial_build [] { + tap begin "local image push + pull + upgrade" + + let td = mktemp -d + cd $td + + do --ignore-errors { podman image rm localhost/bootc o+e>| ignore } + bootc image push + let img = podman image inspect localhost/bootc | from json + + # A simple derived container + "FROM localhost/bootc +RUN echo test content > /usr/share/blah.txt +" | save Dockerfile + # Build it + podman build -t localhost/bootc-derived . + # Just sanity check it + let v = podman run --rm localhost/bootc-derived cat /usr/share/blah.txt | str trim + assert equal $v "test content" + # Now, fetch it back into the bootc storage! + bootc switch --transport containers-storage localhost/bootc-derived + # And reboot into it + tmt-reboot +} + +# The second boot; verify we're in the derived image +def second_boot [] { + assert equal $booted.transport containers-storage + assert equal $booted.image localhost/bootc-derived + let t = open /usr/share/blah.txt | str trim + assert equal $t "test content" + tap ok +} + +def main [] { + # See https://tmt.readthedocs.io/en/stable/stories/features.html#reboot-during-test + match $env.TMT_REBOOT_COUNT? { + null | "0" => initial_build, + "1" => second_boot, + $o => { error make {msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} From 6033b1efb47a08fd0d29bbdc5aedf35250f908b6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 23 Jun 2024 08:47:32 -0400 Subject: [PATCH 2/3] tests/booted: Also verify karg changes We might as well roll in a bunch of things into this "local container build changes" test. I wanted to get some coverage for kargs, so this adds some. Signed-off-by: Colin Walters --- .../booted/002-test-image-pushpull-upgrade.nu | 83 +++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/tests/booted/002-test-image-pushpull-upgrade.nu b/tests/booted/002-test-image-pushpull-upgrade.nu index 3d9e3bbf..f73ccc66 100644 --- a/tests/booted/002-test-image-pushpull-upgrade.nu +++ b/tests/booted/002-test-image-pushpull-upgrade.nu @@ -2,14 +2,27 @@ # bootc image push # podman build # bootc switch +# +# Then another build, and reboot into verifying that use std assert use tap.nu +const kargsv0 = ["testarg=foo", "othertestkarg", "thirdkarg=bar"] +const kargsv1 = ["testarg=foo", "thirdkarg=baz"] +let removed = ($kargsv0 | filter { not ($in in $kargsv1) }) + # This code runs on *each* boot. # Here we just capture information. bootc status let st = bootc status --json | from json -let booted = $st.status.booted.image.image +let booted = $st.status.booted.image + +# Parse the kernel commandline into a list. +# This is not a proper parser, but good enough +# for what we need here. +def parse_cmdline [] { + open /proc/cmdline | str trim | split row " " +} # Run on the first boot def initial_build [] { @@ -22,8 +35,11 @@ def initial_build [] { bootc image push let img = podman image inspect localhost/bootc | from json - # A simple derived container + mkdir usr/lib/bootc/kargs.d + { kargs: $kargsv0 } | to toml | save usr/lib/bootc/kargs.d/05-testkargs.toml + # A simple derived container that adds a file, but also injects some kargs "FROM localhost/bootc +COPY usr/ /usr/ RUN echo test content > /usr/share/blah.txt " | save Dockerfile # Build it @@ -39,10 +55,66 @@ RUN echo test content > /usr/share/blah.txt # The second boot; verify we're in the derived image def second_boot [] { - assert equal $booted.transport containers-storage - assert equal $booted.image localhost/bootc-derived + print "verifying second boot" + # booted from the local container storage and image + assert equal $booted.image.transport containers-storage + assert equal $booted.image.image localhost/bootc-derived + # We wrote this file let t = open /usr/share/blah.txt | str trim assert equal $t "test content" + + # Verify we have updated kargs + let cmdline = parse_cmdline + print $"cmdline=($cmdline)" + for x in $kargsv0 { + print $"verifying karg: ($x)" + assert ($x in $cmdline) + } + + # Now do another build where we drop one of the kargs + let td = mktemp -d + cd $td + + mkdir usr/lib/bootc/kargs.d + { kargs: $kargsv1 } | to toml | save usr/lib/bootc/kargs.d/05-testkargs.toml + "FROM localhost/bootc +COPY usr/ /usr/ +RUN echo test content2 > /usr/share/blah.txt +" | save Dockerfile + # Build it + podman build -t localhost/bootc-derived . + let booted_digest = $booted.imageDigest + print booted_digest = $booted_digest + # We should already be fetching updates from container storage + bootc upgrade + # Verify we staged an update + let st = bootc status --json | from json + let staged_digest = $st.status.staged.image.imageDigest + assert ($booted_digest != $staged_digest) + # And reboot into the upgrade + tmt-reboot +} + +# Check we have the updated kargs +def third_boot [] { + print "verifying third boot" + assert equal $booted.image.transport containers-storage + assert equal $booted.image.image localhost/bootc-derived + let t = open /usr/share/blah.txt | str trim + assert equal $t "test content2" + + # Verify we have updated kargs + let cmdline = parse_cmdline + print $"cmdline=($cmdline)" + for x in $kargsv1 { + print $"Verifying karg ($x)" + assert ($x in $cmdline) + } + # And the kargs that should be removed are gone + for x in $removed { + assert not ($removed in $cmdline) + } + tap ok } @@ -51,6 +123,7 @@ def main [] { match $env.TMT_REBOOT_COUNT? { null | "0" => initial_build, "1" => second_boot, - $o => { error make {msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + "2" => third_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, } } From 83a6a1525ccb3c76ef6fecbbfa98ce770d4fce34 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 27 Jun 2024 11:13:50 -0400 Subject: [PATCH 3/3] cli: Rename to `image copy-to-storage` Drop the transport support; we only support copying to `containers-storage`. Anyone who wants to push directly to a remote registry can do so by first copying to containers-storage and then pushing via e.g. `podman` or `skopeo`. Signed-off-by: Colin Walters --- lib/src/cli.rs | 33 +++---------------- lib/src/image.rs | 8 ++--- .../booted/002-test-image-pushpull-upgrade.nu | 4 +-- 3 files changed, 10 insertions(+), 35 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index e193049b..ccfb87a9 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -199,25 +199,13 @@ pub(crate) enum ImageOpts { /// /// Note that these are distinct from images stored via e.g. `podman`. List, - /// Copy a container image from the bootc storage to a target. - /// - /// ## Copying the booted container image to containers-storage: (podman) + /// Copy a container image from the bootc storage to `containers-storage:`. /// /// The source and target are both optional; if both are left unspecified, - /// via a simple invocation of `bootc image push`, then the default is to + /// via a simple invocation of `bootc image copy-to-storage`, then the default is to /// push the currently booted image to `containers-storage` (as used by podman, etc.) /// and tagged with the image name `localhost/bootc`, /// - /// ## Copying the booted container image to a remote registry - /// - /// Aside from the special case above, default transport is `registry`. This - /// means that an invocation of - /// - /// `bootc image push --target quay.io/example/someimage:latest` will push the - /// booted container image to the target registry. This will be done via an - /// invocation equivalent to `skopeo copy`, and hence the defaults for that apply. - /// For example, the default registry authentication file applies. - /// /// ## Copying a non-default container image /// /// It is also possible to copy an image other than the currently booted one by @@ -227,13 +215,7 @@ pub(crate) enum ImageOpts { /// /// At the current time there is no explicit support for pulling images other than indirectly /// via e.g. `bootc switch` or `bootc upgrade`. - Push { - /// The transport; e.g. oci, oci-archive, containers-storage. Defaults to `registry`. - /// - /// For more information, see `man containers-transports`. - #[clap(long, default_value = "registry")] - transport: String, - + CopyToStorage { #[clap(long)] /// The source image; if not specified, the booted image will be used. source: Option, @@ -794,13 +776,8 @@ async fn run_from_opt(opt: Opt) -> Result<()> { }, Opt::Image(opts) => match opts { ImageOpts::List => crate::image::list_entrypoint().await, - ImageOpts::Push { - transport, - source, - target, - } => { - let transport = Transport::try_from(transport.as_str())?; - crate::image::push_entrypoint(transport, source.as_deref(), target.as_deref()).await + ImageOpts::CopyToStorage { source, target } => { + crate::image::push_entrypoint(source.as_deref(), target.as_deref()).await } }, #[cfg(feature = "install")] diff --git a/lib/src/image.rs b/lib/src/image.rs index d79ed605..4fd7cd9b 100644 --- a/lib/src/image.rs +++ b/lib/src/image.rs @@ -22,12 +22,10 @@ pub(crate) async fn list_entrypoint() -> Result<()> { Ok(()) } +/// Implementation of `bootc image push-to-storage`. #[context("Pushing image")] -pub(crate) async fn push_entrypoint( - transport: Transport, - source: Option<&str>, - target: Option<&str>, -) -> Result<()> { +pub(crate) async fn push_entrypoint(source: Option<&str>, target: Option<&str>) -> Result<()> { + let transport = Transport::ContainerStorage; let sysroot = crate::cli::get_locked_sysroot().await?; let repo = &sysroot.repo(); diff --git a/tests/booted/002-test-image-pushpull-upgrade.nu b/tests/booted/002-test-image-pushpull-upgrade.nu index f73ccc66..9c399118 100644 --- a/tests/booted/002-test-image-pushpull-upgrade.nu +++ b/tests/booted/002-test-image-pushpull-upgrade.nu @@ -1,5 +1,5 @@ # This test does: -# bootc image push +# bootc image copy-to-storage # podman build # bootc switch # @@ -32,7 +32,7 @@ def initial_build [] { cd $td do --ignore-errors { podman image rm localhost/bootc o+e>| ignore } - bootc image push + bootc image copy-to-storage let img = podman image inspect localhost/bootc | from json mkdir usr/lib/bootc/kargs.d