lf-edge · eriknordmark · Dec 13, 2022 · Dec 7, 2022
@@ -138,6 +138,8 @@ ROOTFS=$(INSTALLER)/rootfs
 ROOTFS_FULL_NAME=$(INSTALLER)/rootfs-$(ROOTFS_VERSION)
 ROOTFS_COMPLETE=$(ROOTFS_FULL_NAME)-%-$(ZARCH).$(ROOTFS_FORMAT)
 ROOTFS_IMG=$(ROOTFS).img
+# ROOTFS_TAR is in BUILD_DIR, not installer, so it does not get installed
+ROOTFS_TAR=$(BUILD_DIR)/rootfs.tar
 CONFIG_IMG=$(INSTALLER)/config.img
 INITRD_IMG=$(INSTALLER)/initrd.img
 INSTALLER_IMG=$(INSTALLER)/installer.img
@@ -270,6 +272,8 @@ RESCAN_DEPS=FORCE
 # set FORCE_BUILD to --force to enforce rebuild
 FORCE_BUILD=
 
+SYFT_VERSION:=v0.62.3
+
 # we use the following block to assign correct tag to the Docker registry artifact
 ifeq ($(LINUXKIT_PKG_TARGET),push)
   # only builds from master branch are allowed to be called snapshots
@@ -525,7 +529,9 @@ installer-img: $(INSTALLER_IMG)
 kernel: $(KERNEL_IMG)
 config: $(CONFIG_IMG)		; $(QUIET): "$@: Succeeded, CONFIG_IMG=$(CONFIG_IMG)"
 ssh-key: $(SSH_KEY)
-rootfs: $(ROOTFS_IMG) current
+rootfs: $(ROOTFS_TAR) $(ROOTFS_IMG) current
+rootfs.tar: $(ROOTFS_TAR)
+rootfstar: $(ROOTFS_TAR)
 live: $(LIVE_IMG) $(BIOS_IMG) current	; $(QUIET): "$@: Succeeded, LIVE_IMG=$(LIVE_IMG)"
 live-%: $(LIVE).%		; $(QUIET): "$@: Succeeded, LIVE=$(LIVE)"
 installer: $(INSTALLER_IMG)
@@ -549,9 +555,14 @@ $(ROOTFS)-%.img: $(ROOTFS_IMG)
 	@rm -f $@ && ln -s $(notdir $<) $@
 	$(QUIET): $@: Succeeded
 
-$(ROOTFS_IMG): images/rootfs-$(HV).yml | $(INSTALLER)
+$(ROOTFS_TAR): images/rootfs-$(HV).yml | $(INSTALLER)
+	$(QUIET): $@: Begin
+	./tools/makerootfs.sh tar -y $< -t $@ -a $(ZARCH)
+	$(QUIET): $@: Succeeded
+
+$(ROOTFS_IMG): $(ROOTFS_TAR) | $(INSTALLER)
 	$(QUIET): $@: Begin
-	./tools/makerootfs.sh $< $@ $(ROOTFS_FORMAT) $(ZARCH)
+	./tools/makerootfs.sh imagefromtar -t $(ROOTFS_TAR) -i $@ -f $(ROOTFS_FORMAT) -a $(ZARCH)
 	@echo "size of $@ is $$(wc -c < "$@")B"
 	@[ $$(wc -c < "$@") -gt $$(( 250 * 1024 * 1024 )) ] && \
 	        echo "ERROR: size of $@ is greater than 250MB (bigger than allocated partition)" && exit 1 || :

@@ -154,23 +154,44 @@ EVE builds one of two bootable images using linuxkit, depending on if you are bu
 
 For a live bootable image, named `live.img`, we create the following dependencies in order:
 
-1. `rootfs.img`
-2. `config.img`
-3. `live.raw`
-4. `live.qcow2`
+1. `rootfs.tar`
+2. `rootfs.img`
+3. `config.img`
+4. `live.raw`
+5. `live.qcow2`
+
+#### `rootfs.tar`
+
+This is the rootfs filesystem in a tar file.
+It is a temporary artifact which is used as input into security and bill-of-materials scanners,
+and is then consumed to create other artifacts, and then can be removed. To build it:
+
+1. Verify the existence of the linuxkit builder configuration file `images/rootfs.yml`. See notes on [generating yml](#generating-yml).
+1. Call `makerootfs.sh tar -y images/rootfs.yml -t path/to/rootfs.tar [-a <arch>]`, which will build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/rootfs.yml` as the configuration file, saving the output to `path/to/rootfs.tar`.
+
+When done with any later builds, you can remove the temporary artifact `path/to/rootfs.tar`.
+
+When run with `make rootfstar` or `make rootfs`, this will build the rootfs tar file:
+
+* For the default architecture;
+* Saving the tar file to `dist/<arch>/<path-from-commit-and-version-and-date>/`
 
 #### rootfs.img
 
 `rootfs.img` is a bootable root filesystem. To build it:
 
-1. Verify the existence of the linuxkit builder configuration file `images/rootfs.yml`. See notes on [generating yml](#generating-yml).
-2. Call `makerootfs.sh images/rootfs.yml rootfs.img <format> <arch>`, which will:
-    1. Build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/rootfs.yml` as the configuration file.
-    2. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
-    3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cf`, builds it into an ext4 filesystem image, and streams it to stdout.
-    4. Direct the output to the targeted filename, in this case `rootfs.img`.
+1. Verify the existence of the rootfs tar file from the previous stage.
+1. Call `makerootfs.sh imagefromtar -i path/to/rootfs.img -t path/to/rootfs.tar -f <format>`, which will:
+    1. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
+    2. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cfg`, builds it into an ext4 filesystem image, and streams it to stdout.
+    3. Direct the output to the targeted filename, in this case `path/to/rootfs.img`.
+
+We now have `path/to/rootfs.img`.
+
+If you will not need any of the interim steps, you can run `makerootfs.sh image -y images/rootfs.yml -f <format> -a <arch> -i path/to/rootfs.img`, which combines:
 
-We now have `rootfs.img`.
+1. Creating the tar
+1. Creating the disk image
 
 #### config.img
 
@@ -253,7 +274,7 @@ To build `rootfs_installer.img`:
 2. Call `makerootfs.sh images/installer.yml rootfs_installer.img <format> <arch>`, which will:
     1. Build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/installer.yml` as the configuration file..
     2. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
-    3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cf`, builds it into an ext4 filesystem image, and streams it to stdout.
+    3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cfg`, builds it into an ext4 or squasfhs filesystem image, and streams it to stdout.
     4. Direct the output to the targeted filename, in this case `rootfs_installer.img`.
 
 #### Installer: installer.raw

@@ -1,30 +1,122 @@
 #!/bin/bash
 # Usage:
 #
-#     ./makerootfs.sh <image.yml> <output rootfs image> <fs> <arch>
+#     ./makerootfs.sh mode [-y <image.yml>] [-i <output rootfs image>] [-t <tar file>] [-f <filesystem format>] [-a <arch>]
 # <fs> defaults to squash
 # <arch> defaults to the current machine architecture
 
 set -e
 set -o pipefail
 
+# mode 3 - generate image from yml
+do_image() {
+  # shellcheck disable=SC2166
+  if [ -z "$ymlfile" -o -z "$imgfile" -o -n "$tarfile" ]; then
+    echo "must supply ymlfile and imgfile and not tarfile" >&2
+    help
+  fi
+  # did we specify an architecture?
+  ARCHARG=""
+  if [ -n "$arch" ]; then
+    ARCHARG="-arch ${arch}"
+  fi
+  : > "$IMAGE"
+  # shellcheck disable=SC2086
+  linuxkit build -docker ${ARCHARG} -o - "$ymlfile" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
+}
+
+# mode 1 - generate tarfile from yml and save
+do_tar() {
+  # shellcheck disable=SC2166
+  if [ -z "$ymlfile" -o -z "$tarfile" -o -n "$imgfile" ]; then
+    echo "must supply ymlfile and tarfile and not imgfile" >&2
+    help
+  fi
+  # did we specify an architecture?
+  ARCHARG=""
+  if [ -n "$arch" ]; then
+    ARCHARG="-arch ${arch}"
+  fi
+  # shellcheck disable=SC2086
+  linuxkit build -docker ${ARCHARG} -o "${tarfile}" "$ymlfile"
+}
+
+# mode 2 - generate image from tarfile
+do_imagefromtar() {
+  # shellcheck disable=SC2166
+  if [ -z "$tarfile" -o -z "$imgfile" -o -n "$ymlfile" ]; then
+    echo "must supply tarfile and imgfile and not ymlfile" >&2
+    help
+  fi
+  : > "$IMAGE"
+  # shellcheck disable=SC2002
+  cat "${tarfile}" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
+}
+
+bail() {
+  echo "$@" >&2
+  help
+}
+
+
+# no mode we recognize
+help() {
+  echo "Usage: $0 <mode> [-y <image.yml>] [-i <output rootfs image>] [-t <tarfile>] [-f {ext4|squash}] [-a <arch>]" >&2
+  echo "must be one of the following modes:" >&2
+  echo "  generate final image from yml:" >&2
+  echo "    $0 image [-y <image.yml>] [-i <output rootfs image>] [-f {ext4|squash}] [-a <arch>]" >&2
+  echo "  generate tar from yml:" >&2
+  echo "    $0 tar [-y <image.yml>] [-t <output tarfile>] [-a <arch>]" >&2
+  echo "  generate final image from tar:" >&2
+  echo "    $0 imagefromtar [-i <output rootfs image>] [-f {ext4|squash}] [-t <input tarfile>]" >&2
+  exit 1
+}
+
+# there are 3 modes we can run in:
+# 1. generate tarfile from yml and save
+# 2. generate image from tarfile
+# 3. generate image from yml
+mode="$1"
+shift
+
+unset tarfile imgfile arch format ymlfile
+while getopts "t:i:a:f:y:h" o
+do
+  case $o in
+    t)
+      tarfile=$OPTARG
+      ;;
+    i)
+      imgfile=$OPTARG
+      ;;
+    a)
+      arch=$OPTARG
+      ;;
+    f)
+      format=$OPTARG
+      ;;
+    y)
+      ymlfile=$OPTARG
+      ;;
+    h)
+      help
+      ;;
+    *)
+      ;;
+  esac
+done
+
+[ -z "$format" ] && format="squash"
+
 EVE="$(cd "$(dirname "$0")" && pwd)/../"
 PATH="$EVE/build-tools/bin:$PATH"
-MKROOTFS_TAG="$(linuxkit pkg show-tag "$EVE/pkg/mkrootfs-${3:-squash}")"
-YMLFILE="$1"
-IMAGE="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"
-
-if [ ! -f "$YMLFILE" ] || [ $# -lt 2 ]; then
-    echo "Usage: $0 <image.yml> <output rootfs image> [{ext4|squash}]"
-    exit 1
-fi
-
-# did we specify an architecture?
-ARCH="$4"
-ARCHARG=""
-if [ -n "$ARCH" ]; then
-  ARCHARG="-arch ${ARCH}"
-fi
-
-: > "$IMAGE"
-linuxkit build -docker ${ARCHARG} -o - "$YMLFILE" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
+MKROOTFS_TAG="$(linuxkit pkg show-tag "$EVE/pkg/mkrootfs-${format}")"
+IMAGE="$(cd "$(dirname "$imgfile")" && pwd)/$(basename "$imgfile")"
+
+action="do_${mode}"
+#shellcheck disable=SC2039
+command -V "$action" > /dev/null 2>&1 || bail "Error: unsupported command '$mode'."
+
+# Do. Or do not. There is no try.
+"$action" "$@"
+