fs: Add code to import from a filesystem

src/fs.rs contains code for writing the in-memory filesystem tree to a
directory on disk, so let's add the other direction: converting an
on-disk directory to an in-memory filesystem tree.  This will let us
scan container images from inside containers.  This is necessary because
we can't get access to the OCI layer tarballs during a container build
(even from a later stage in a multi-stage build) but we can bindmount
the root filesystem.

See containers/buildah#5837

With our recent changes to how we handle metadata on the root directory
we should now be producing the same image on the inside and the outside,
which gives us a nice way to produce a UKI with a built-in `composefs=`
command-line parameter.

Add a new 'unified' example.  This does the container build as a single
`podman build` command with no special arguments.

Closes #34

examples/unified/.gitignore

@@ -0,0 +1,5 @@
+/cfsctl
+/extra/usr/lib/dracut/modules.d/37composefs/composefs-pivot-sysroot
+/fix-verity.efi
+/image.qcow2
+/tmp/

examples/unified/Containerfile

@@ -0,0 +1,48 @@
+# Need 6.12 kernel from rawhide
+FROM fedora:rawhide AS base
+COPY extra /
+COPY cfsctl /usr/bin
+RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
+    set -eux
+
+    # we should install kernel-modules here, but can't
+    # because it'll pull in the entire kernel with it
+    # it seems to work fine for now....
+    dnf --setopt keepcache=1 install -y \
+        composefs \
+        dosfstools \
+        policycoreutils-python-utils \
+        selinux-policy-targeted \
+        skopeo \
+        strace \
+        systemd \
+        util-linux
+    systemctl enable systemd-networkd
+    semanage permissive -a systemd_gpt_generator_t  # for volatile-root workaround
+    passwd -d root
+    mkdir /sysroot
+EOF
+
+FROM base AS kernel
+RUN --mount=type=bind,from=base,target=/mnt/base <<EOF
+    set -eux
+
+    mkdir -p /tmp/sysroot/composefs
+    COMPOSEFS_FSVERITY="$(cfsctl --repo /tmp/sysroot create-image /mnt/base)"
+
+    mkdir -p /etc/kernel /etc/dracut.conf.d
+    echo "composefs=${COMPOSEFS_FSVERITY} rw" > /etc/kernel/cmdline
+EOF
+RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
+    # systemd-boot-unsigned: ditto
+    # btrfs-progs: dracut wants to include this in the initramfs
+    # ukify: dracut doesn't want to take our cmdline args?
+    dnf --setopt keepcache=1 install -y kernel btrfs-progs systemd-boot-unsigned systemd-ukify
+EOF
+
+# This could (better?) be done from cfsctl...
+FROM base AS bootable
+COPY --from=kernel /boot /composefs-meta/boot
+# RUN rm -rf /composefs-meta
+# RUN commands touch /run unfortunately
+COPY empty /.wh.composefs-meta

examples/unified/build

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -eux
+
+cd "${0%/*}"
+
+cargo build --release
+
+cp ../../target/release/cfsctl .
+cp ../../target/release/composefs-pivot-sysroot extra/usr/lib/dracut/modules.d/37composefs/
+CFSCTL='./cfsctl --repo tmp/sysroot/composefs'
+
+rm -rf tmp
+mkdir -p tmp/sysroot/composefs tmp/sysroot/var
+
+# mkdir tmp/internal-sysroot  # for debugging
+# podman build -v $(pwd)/tmp/internal-sysroot:/tmp/sysroot:z,U --iidfile=tmp/iid "$@" .
+#
+podman build --iidfile=tmp/iid "$@" .
+
+IMAGE_ID="$(sed s/sha256:// tmp/iid)"
+podman save --format oci-archive -o tmp/final.tar "${IMAGE_ID}"
+${CFSCTL} oci pull oci-archive:tmp/final.tar
+IMAGE_FSVERITY="$(${CFSCTL} oci create-image "${IMAGE_ID}")"
+
+mkdir -p tmp/efi/loader
+echo 'timeout 3' > tmp/efi/loader/loader.conf
+mkdir -p tmp/efi/EFI/BOOT tmp/efi/EFI/systemd
+cp /usr/lib/systemd/boot/efi/systemd-bootx64.efi tmp/efi/EFI/systemd
+cp /usr/lib/systemd/boot/efi/systemd-bootx64.efi tmp/efi/EFI/BOOT/BOOTX64.EFI
+${CFSCTL} oci prepare-boot "${IMAGE_ID}" tmp/efi
+
+fakeroot ./make-image
+qemu-img convert -f raw tmp/image.raw -O qcow2 image.qcow2
+./fix-verity image.qcow2  # https://github.com/tytso/e2fsprogs/issues/201

examples/unified/extra/etc/resolv.conf

@@ -0,0 +1 @@
+../run/systemd/resolve/stub-resolv.conf

examples/unified/extra/usr/lib/dracut/dracut.conf.d/37composefs.conf

@@ -0,0 +1,6 @@
+# we want to make sure the virtio disk drivers get included
+hostonly=no
+
+# we need to force these in via the initramfs because we don't have modules in
+# the base image
+force_drivers+=" virtio_net vfat "

examples/unified/extra/usr/lib/dracut/modules.d/37composefs/composefs-pivot-sysroot.service

@@ -0,0 +1,34 @@
+# Copyright (C) 2013 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <https://www.gnu.org/licenses/>.
+
+[Unit]
+DefaultDependencies=no
+ConditionKernelCommandLine=composefs
+ConditionPathExists=/etc/initrd-release
+After=sysroot.mount
+Requires=sysroot.mount
+Before=initrd-root-fs.target
+Before=initrd-switch-root.target
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/composefs-pivot-sysroot
+StandardInput=null
+StandardOutput=journal
+StandardError=journal+console
+RemainAfterExit=yes

examples/unified/extra/usr/lib/dracut/modules.d/37composefs/module-setup.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/bash
+
+check() {
+    return 0
+}
+
+depends() {
+    return 0
+}
+
+install() {
+    inst \
+        "${moddir}/composefs-pivot-sysroot" /bin/composefs-pivot-sysroot
+    inst \
+        "${moddir}/composefs-pivot-sysroot.service" \
+        "${systemdsystemunitdir}/composefs-pivot-sysroot.service"
+
+    $SYSTEMCTL -q --root "${initdir}" add-wants \
+        'initrd-root-fs.target' 'composefs-pivot-sysroot.service'
+}

examples/unified/extra/usr/lib/kernel/install.conf.d/37composefs.conf

@@ -0,0 +1,2 @@
+layout = uki
+uki_generator = ukify

examples/unified/extra/usr/lib/systemd/network/37-wired.network

@@ -0,0 +1,9 @@
+[Match]
+Type=ether
+
+[Link]
+RequiredForOnline=routable
+
+[Network]
+DHCP=yes
+

examples/unified/extra/usr/lib/systemd/system/systemd-growfs-root.service.d/37-composefs.conf

@@ -0,0 +1,6 @@
+# Make sure we grow the right root filesystem
+
+[Service]
+ExecStart=
+ExecStart=/usr/lib/systemd/systemd-growfs /sysroot
+

examples/unified/fix-verity

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+# workaround for https://github.com/tytso/e2fsprogs/issues/201
+
+set -eux
+
+# We use a custom UKI with an initramfs containing a script that remounts
+# /sysroot read-write and enables fs-verity on all of the objects in
+# /composefs/objects.
+#
+# The first time we're run (or if we are modified) we (re-)generate the UKI.
+# This is done inside of a container (for independence from the host OS).
+
+image_file="$1"
+
+if [ "$0" -nt fix-verity.efi ]; then
+    podman run --rm -i fedora > tmp/fix-verity.efi <<'EOF'
+    set -eux
+
+    cat > /tmp/fix-verity.sh <<'EOS'
+    mount -o remount,rw /sysroot
+    (
+      cd /sysroot/composefs/objects
+      echo >&2 'Enabling fsverity on composefs objects'
+      for i in */*; do
+          fsverity enable $i;
+      done
+      echo >&2 'done!'
+    )
+    umount /sysroot
+    sync
+    poweroff -ff
+EOS
+
+    (
+      dnf --setopt keepcache=1 install -y \
+        kernel binutils systemd-boot-unsigned btrfs-progs fsverity-utils
+      dracut \
+        --uefi \
+        --no-hostonly \
+        --install 'sync fsverity' \
+        --include /tmp/fix-verity.sh /lib/dracut/hooks/pre-pivot/fix-verity.sh \
+        --kver "$(rpm -q kernel-core --qf '%{VERSION}-%{RELEASE}.%{ARCH}')" \
+        --kernel-cmdline="root=PARTLABEL=root-x86-64 console=ttyS0" \
+        /tmp/fix-verity.efi
+    ) >&2
+
+    cat /tmp/fix-verity.efi
+EOF
+    mv tmp/fix-verity.efi fix-verity.efi
+fi
+
+qemu-system-x86_64 \
+    -nographic \
+    -m 4096 \
+    -enable-kvm \
+    -bios /usr/share/edk2/ovmf/OVMF_CODE.fd \
+    -drive file="$1",if=virtio,media=disk \
+    -kernel fix-verity.efi

examples/unified/make-image

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -eux
+
+chown -R 0:0 tmp/sysroot
+chcon -R system_u:object_r:usr_t:s0 tmp/sysroot/composefs
+chcon system_u:object_r:var_t:s0 tmp/sysroot/var
+
+> tmp/image.raw
+SYSTEMD_REPART_MKFS_OPTIONS_EXT4='-O verity' \
+    systemd-repart \
+        --empty=require \
+        --size=auto \
+        --dry-run=no \
+        --no-pager \
+        --offline=yes \
+        --root=tmp \
+        --definitions=repart.d \
+        tmp/image.raw

examples/unified/repart.d/01-esp.conf

@@ -0,0 +1,6 @@
+[Partition]
+Type=esp
+Format=vfat
+CopyFiles=/efi:/
+SizeMinBytes=512M
+SizeMaxBytes=512M

examples/unified/repart.d/02-sysroot.conf

@@ -0,0 +1,6 @@
+[Partition]
+Type=root
+Format=ext4
+SizeMinBytes=10G
+SizeMaxBytes=10G
+CopyFiles=/sysroot:/

examples/unified/run

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -eux
+
+cd "${0%/*}"
+
+qemu-system-x86_64 \
+    -m 4096 \
+    -enable-kvm \
+    -bios /usr/share/edk2/ovmf/OVMF_CODE.fd \
+    -drive file=image.qcow2,if=virtio,cache=unsafe \
+    -nic user,model=virtio-net-pci

src/bin/cfsctl.rs

@@ -73,7 +73,9 @@ enum Command {
     /// Perform garbage collection
     GC,
     /// Imports a composefs image (unsafe!)
-    ImportImage { reference: String },
+    ImportImage {
+        reference: String,
+    },
     /// Commands for dealing with OCI layers
     Oci {
         #[clap(subcommand)]
@@ -86,6 +88,12 @@ enum Command {
         /// the mountpoint
         mountpoint: String,
     },
+    CreateImage {
+        path: PathBuf,
+    },
+    CreateDumpfile {
+        path: PathBuf,
+    },
 }
 
 fn main() -> Result<()> {
@@ -165,6 +173,13 @@ fn main() -> Result<()> {
                 oci::prepare_boot(&repo, name, None, &output)?;
             }
         },
+        Command::CreateImage { ref path } => {
+            let image_id = composefs::fs::create_image(path, Some(&repo))?;
+            println!("{}", hex::encode(image_id));
+        }
+        Command::CreateDumpfile { ref path } => {
+            composefs::fs::create_dumpfile(path)?;
+        }
         Command::Mount { name, mountpoint } => {
             repo.mount(&name, &mountpoint)?;
         }