Skip to content

Commit

Permalink
separate steps for generating rootfs
Browse files Browse the repository at this point in the history
Signed-off-by: Avi Deitcher <avi@deitcher.net>
  • Loading branch information
deitch committed Dec 12, 2022
1 parent 4c1d24d commit 34d149a
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 34 deletions.
17 changes: 14 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ ROOTFS=$(INSTALLER)/rootfs
ROOTFS_FULL_NAME=$(INSTALLER)/rootfs-$(ROOTFS_VERSION)
ROOTFS_COMPLETE=$(ROOTFS_FULL_NAME)-%-$(ZARCH).$(ROOTFS_FORMAT)
ROOTFS_IMG=$(ROOTFS).img
# ROOTFS_TAR is in BUILD_DIR, not installer, so it does not get installed
ROOTFS_TAR=$(BUILD_DIR)/rootfs.tar
CONFIG_IMG=$(INSTALLER)/config.img
INITRD_IMG=$(INSTALLER)/initrd.img
INSTALLER_IMG=$(INSTALLER)/installer.img
Expand Down Expand Up @@ -270,6 +272,8 @@ RESCAN_DEPS=FORCE
# set FORCE_BUILD to --force to enforce rebuild
FORCE_BUILD=

SYFT_VERSION:=v0.62.3

# we use the following block to assign correct tag to the Docker registry artifact
ifeq ($(LINUXKIT_PKG_TARGET),push)
# only builds from master branch are allowed to be called snapshots
Expand Down Expand Up @@ -525,7 +529,9 @@ installer-img: $(INSTALLER_IMG)
kernel: $(KERNEL_IMG)
config: $(CONFIG_IMG) ; $(QUIET): "$@: Succeeded, CONFIG_IMG=$(CONFIG_IMG)"
ssh-key: $(SSH_KEY)
rootfs: $(ROOTFS_IMG) current
rootfs: $(ROOTFS_TAR) $(ROOTFS_IMG) current
rootfs.tar: $(ROOTFS_TAR)
rootfstar: $(ROOTFS_TAR)
live: $(LIVE_IMG) $(BIOS_IMG) current ; $(QUIET): "$@: Succeeded, LIVE_IMG=$(LIVE_IMG)"
live-%: $(LIVE).% ; $(QUIET): "$@: Succeeded, LIVE=$(LIVE)"
installer: $(INSTALLER_IMG)
Expand All @@ -549,9 +555,14 @@ $(ROOTFS)-%.img: $(ROOTFS_IMG)
@rm -f $@ && ln -s $(notdir $<) $@
$(QUIET): $@: Succeeded

$(ROOTFS_IMG): images/rootfs-$(HV).yml | $(INSTALLER)
$(ROOTFS_TAR): images/rootfs-$(HV).yml | $(INSTALLER)
$(QUIET): $@: Begin
./tools/makerootfs.sh tar -y $< -t $@ -a $(ZARCH)
$(QUIET): $@: Succeeded

$(ROOTFS_IMG): $(ROOTFS_TAR) | $(INSTALLER)
$(QUIET): $@: Begin
./tools/makerootfs.sh $< $@ $(ROOTFS_FORMAT) $(ZARCH)
./tools/makerootfs.sh imagefromtar -t $(ROOTFS_TAR) -i $@ -f $(ROOTFS_FORMAT) -a $(ZARCH)
@echo "size of $@ is $$(wc -c < "$@")B"
@[ $$(wc -c < "$@") -gt $$(( 250 * 1024 * 1024 )) ] && \
echo "ERROR: size of $@ is greater than 250MB (bigger than allocated partition)" && exit 1 || :
Expand Down
45 changes: 33 additions & 12 deletions docs/BUILD.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,23 +154,44 @@ EVE builds one of two bootable images using linuxkit, depending on if you are bu

For a live bootable image, named `live.img`, we create the following dependencies in order:

1. `rootfs.img`
2. `config.img`
3. `live.raw`
4. `live.qcow2`
1. `rootfs.tar`
2. `rootfs.img`
3. `config.img`
4. `live.raw`
5. `live.qcow2`

#### `rootfs.tar`

This is the rootfs filesystem in a tar file.
It is a temporary artifact which is used as input into security and bill-of-materials scanners,
and is then consumed to create other artifacts, and then can be removed. To build it:

1. Verify the existence of the linuxkit builder configuration file `images/rootfs.yml`. See notes on [generating yml](#generating-yml).
1. Call `makerootfs.sh tar -y images/rootfs.yml -t path/to/rootfs.tar [-a <arch>]`, which will build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/rootfs.yml` as the configuration file, saving the output to `path/to/rootfs.tar`.

When done with any later builds, you can remove the temporary artifact `path/to/rootfs.tar`.

When run with `make rootfstar` or `make rootfs`, this will build the rootfs tar file:

* For the default architecture;
* Saving the tar file to `dist/<arch>/<path-from-commit-and-version-and-date>/`

#### rootfs.img

`rootfs.img` is a bootable root filesystem. To build it:

1. Verify the existence of the linuxkit builder configuration file `images/rootfs.yml`. See notes on [generating yml](#generating-yml).
2. Call `makerootfs.sh images/rootfs.yml rootfs.img <format> <arch>`, which will:
1. Build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/rootfs.yml` as the configuration file.
2. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cf`, builds it into an ext4 filesystem image, and streams it to stdout.
4. Direct the output to the targeted filename, in this case `rootfs.img`.
1. Verify the existence of the rootfs tar file from the previous stage.
1. Call `makerootfs.sh imagefromtar -i path/to/rootfs.img -t path/to/rootfs.tar -f <format>`, which will:
1. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
2. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cfg`, builds it into an ext4 filesystem image, and streams it to stdout.
3. Direct the output to the targeted filename, in this case `path/to/rootfs.img`.

We now have `path/to/rootfs.img`.

If you will not need any of the interim steps, you can run `makerootfs.sh image -y images/rootfs.yml -f <format> -a <arch> -i path/to/rootfs.img`, which combines:

We now have `rootfs.img`.
1. Creating the tar
1. Creating the disk image

#### config.img

Expand Down Expand Up @@ -253,7 +274,7 @@ To build `rootfs_installer.img`:
2. Call `makerootfs.sh images/installer.yml rootfs_installer.img <format> <arch>`, which will:
1. Build an image for the target architecture `<arch>` using `linuxkit` with a tar output format using `images/installer.yml` as the configuration file..
2. Pipe the contents of the tar image to a docker container from either `mkrootfs-ext4` or `mkrootfs-squash`, depending on desired output format.
3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cf`, builds it into an ext4 filesystem image, and streams it to stdout.
3. `mkrootfs-*` takes the contents of the tar image, modifies `grub.cfg`, builds it into an ext4 or squasfhs filesystem image, and streams it to stdout.
4. Direct the output to the targeted filename, in this case `rootfs_installer.img`.

#### Installer: installer.raw
Expand Down
130 changes: 111 additions & 19 deletions tools/makerootfs.sh
Original file line number Diff line number Diff line change
@@ -1,30 +1,122 @@
#!/bin/bash
# Usage:
#
# ./makerootfs.sh <image.yml> <output rootfs image> <fs> <arch>
# ./makerootfs.sh mode [-y <image.yml>] [-i <output rootfs image>] [-t <tar file>] [-f <filesystem format>] [-a <arch>]
# <fs> defaults to squash
# <arch> defaults to the current machine architecture

set -e
set -o pipefail

# mode 3 - generate image from yml
do_image() {
# shellcheck disable=SC2166
if [ -z "$ymlfile" -o -z "$imgfile" -o -n "$tarfile" ]; then
echo "must supply ymlfile and imgfile and not tarfile" >&2
help
fi
# did we specify an architecture?
ARCHARG=""
if [ -n "$arch" ]; then
ARCHARG="-arch ${arch}"
fi
: > "$IMAGE"
# shellcheck disable=SC2086
linuxkit build -docker ${ARCHARG} -o - "$ymlfile" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
}

# mode 1 - generate tarfile from yml and save
do_tar() {
# shellcheck disable=SC2166
if [ -z "$ymlfile" -o -z "$tarfile" -o -n "$imgfile" ]; then
echo "must supply ymlfile and tarfile and not imgfile" >&2
help
fi
# did we specify an architecture?
ARCHARG=""
if [ -n "$arch" ]; then
ARCHARG="-arch ${arch}"
fi
# shellcheck disable=SC2086
linuxkit build -docker ${ARCHARG} -o "${tarfile}" "$ymlfile"
}

# mode 2 - generate image from tarfile
do_imagefromtar() {
# shellcheck disable=SC2166
if [ -z "$tarfile" -o -z "$imgfile" -o -n "$ymlfile" ]; then
echo "must supply tarfile and imgfile and not ymlfile" >&2
help
fi
: > "$IMAGE"
# shellcheck disable=SC2002
cat "${tarfile}" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
}

bail() {
echo "$@" >&2
help
}


# no mode we recognize
help() {
echo "Usage: $0 <mode> [-y <image.yml>] [-i <output rootfs image>] [-t <tarfile>] [-f {ext4|squash}] [-a <arch>]" >&2
echo "must be one of the following modes:" >&2
echo " generate final image from yml:" >&2
echo " $0 image [-y <image.yml>] [-i <output rootfs image>] [-f {ext4|squash}] [-a <arch>]" >&2
echo " generate tar from yml:" >&2
echo " $0 tar [-y <image.yml>] [-t <output tarfile>] [-a <arch>]" >&2
echo " generate final image from tar:" >&2
echo " $0 imagefromtar [-i <output rootfs image>] [-f {ext4|squash}] [-t <input tarfile>]" >&2
exit 1
}

# there are 3 modes we can run in:
# 1. generate tarfile from yml and save
# 2. generate image from tarfile
# 3. generate image from yml
mode="$1"
shift

unset tarfile imgfile arch format ymlfile
while getopts "t:i:a:f:y:h" o
do
case $o in
t)
tarfile=$OPTARG
;;
i)
imgfile=$OPTARG
;;
a)
arch=$OPTARG
;;
f)
format=$OPTARG
;;
y)
ymlfile=$OPTARG
;;
h)
help
;;
*)
;;
esac
done

[ -z "$format" ] && format="squash"

EVE="$(cd "$(dirname "$0")" && pwd)/../"
PATH="$EVE/build-tools/bin:$PATH"
MKROOTFS_TAG="$(linuxkit pkg show-tag "$EVE/pkg/mkrootfs-${3:-squash}")"
YMLFILE="$1"
IMAGE="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"

if [ ! -f "$YMLFILE" ] || [ $# -lt 2 ]; then
echo "Usage: $0 <image.yml> <output rootfs image> [{ext4|squash}]"
exit 1
fi

# did we specify an architecture?
ARCH="$4"
ARCHARG=""
if [ -n "$ARCH" ]; then
ARCHARG="-arch ${ARCH}"
fi

: > "$IMAGE"
linuxkit build -docker ${ARCHARG} -o - "$YMLFILE" | docker run -i --rm -v /dev:/dev --privileged -v "$IMAGE:/rootfs.img" "${MKROOTFS_TAG}"
MKROOTFS_TAG="$(linuxkit pkg show-tag "$EVE/pkg/mkrootfs-${format}")"
IMAGE="$(cd "$(dirname "$imgfile")" && pwd)/$(basename "$imgfile")"

action="do_${mode}"
#shellcheck disable=SC2039
command -V "$action" > /dev/null 2>&1 || bail "Error: unsupported command '$mode'."

# Do. Or do not. There is no try.
"$action" "$@"

0 comments on commit 34d149a

Please sign in to comment.