diff --git a/.gitignore b/.gitignore index 596e47f158..1683a2d4a1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ yetus-output/ assets/ pkg/kernel/build.yml pkg/new-kernel/build.yml +pkg/kernel/certs/*.pem diff --git a/docs/BUILD.md b/docs/BUILD.md index 3763989870..6ee0f24723 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -506,7 +506,7 @@ docker run -p 8080:80 -p 8125:8125/udp --rm --name statsd graphiteapp/graphite-s While the EVE is running, navigate to `http://IP:8080/dashboard` and find the result under `stats.gauges`. -#### Kernel versioning +#### Reproducible Kernel build and versioning Kernel packages ("pkg/kernel" and "pkg/new-kernel") are configured to produce a bit-by-bit reproducible kernel, to this end a `build.yml` is generated at build time to set the following variables to a static value: @@ -518,7 +518,43 @@ KCONFIG_NOTIMESTAMP=true In addition, both `KBUILD_BUILD_TIMESTAMP` and `SOURCE_DATE_EPOCH` variables are set to the last commit date of the respective package, This configuration results in having a static version string (`/proc/version`) on every build. In case there is uncommitted changes in the kernel(s) directory, kernel gets build normally without any static time. -This process can be used to compare eve images that are build in a trusted environment vs CI, making sure the automated build process is intact and not malicious or compromised. +This process can be used to compare eve images that are build in a trusted environment vs CI, making sure the automated build process is intact and not malicious or compromised. For this purpose, you can use [rootfs-diff.sh](../tools/rootfs-diff.sh) to compare two builds. The script accepts the path of the mounted rootfs.img files (you need to mount a RW overlay on top; check the comments on the script), ideally you should see no output after the diff is finished: + +```bash +$ rootfs-diff.sh /tmp/rfs-one/ tmp/rfs-two/ +[1] Removing the signing key from kernel... +[1] Removing the signature form kernel modules... +[2] Removing the signing key from kernel... +[2] Removing the signature form kernel modules... +[*] Diffing the two rootfs... +$ +``` + +But in case the two builds differ, the script outputs a list of files: + +```bash +$ rootfs-diff.sh /tmp/rfs-one/ tmp/rfs-two/ +[1] Removing the signing key from kernel... +[1] Removing the signature form kernel modules... +[2] Removing the signing key from kernel... +[2] Removing the signature form kernel modules... +[*] Diffing the two rootfs... +Files /tmp/rfs-one/boot/kernel and /tmp/rfs-two/boot/kernel differ +Files /tmp/rfs-one/lib/modules/5.10.121-default/kernel/net/can/can.ko and /tmp/rfs-two/lib/modules/5.10.121-default/kernel/net/can/can.ko differ +$ +``` + +#### Buil-time kernel module signing + +EVE kernel is configured to only load signed kernel modules, the module signing happens automatically during the build with a build-time generated throw-away key. But if you are willing to sign the modules using your own key, generate a key: + +```shell +openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ + -config x509.genkey -outform PEM -out kernel_key.pem \ + -keyout kernel_key.pem +``` + +and place the `.pem` file in the `/eve/pkg/kernel/certs` directory. You can respectively change the `x509.genkey` template too. ## Summary of Build Process diff --git a/pkg/kernel/Dockerfile b/pkg/kernel/Dockerfile index 12fb4281dd..3deaed9e14 100644 --- a/pkg/kernel/Dockerfile +++ b/pkg/kernel/Dockerfile @@ -79,6 +79,7 @@ ENV KERNEL_DEFCONFIG=x86_64_defconfig FROM kernel-target-${TARGETARCH} AS kernel-build COPY /kernel-config/* / +COPY /certs /certs COPY /patches-5.10.x /patches-5.10.x COPY /patches-zfs-2.1.2 /patches-zfs-2.1.2 @@ -113,6 +114,15 @@ RUN set -e ; KERNEL_SERIES="${KERNEL_VERSION%.*}".x; \ patch -p1 < "$patch"; \ done +# Copy module signing x509 template and key +WORKDIR /linux +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] +RUN cp /certs/x509.genkey /linux/certs/; \ + if find certs/ -name \*.pem | grep . ; then \ + KEY=$(find certs/ -name \*.pem | head -1); \ + cp "${KEY}" /linux/certs/signing_key.pem; \ + fi + # Copy default kconfig and prepare kbuild # hadolint ignore=SC2086 RUN KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \ @@ -216,6 +226,26 @@ RUN make -j "$(getconf _NPROCESSORS_ONLN)" CROSS_COMPILE="${CROSS_COMPILE_ENV}" # Strip at least some of the modules to conserve space RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then "${CROSS_COMPILE_ENV}strip" --strip-debug `find /tmp/kernel-modules/lib/modules -name \*.ko` ;fi +# Resign stripped kernel modules again +RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then \ + KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \ + if grep -q "^CONFIG_MODULE_SIG_FORCE=y" "${KERNEL_DEF_CONF}" ;then \ + make INSTALL_MOD_PATH=/tmp/kernel-modules modules_sign ; \ + fi \ +fi + +# Sign out-of-tree module(s) +WORKDIR /linux +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] +# hadolint ignore=SC2086,SC2046 +RUN KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \ + if grep -q "^CONFIG_MODULE_SIG_FORCE=y" "${KERNEL_DEF_CONF}" ;then \ + SIG_HASH=$(sed -n '/^CONFIG_MODULE_SIG_HASH=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \ + SIG_KEY_SRCPREFIX=$(sed -n '/^MODULE_SIG_KEY_SRCPREFIX=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \ + SIG_KEY=$(sed -n '/^CONFIG_MODULE_SIG_KEY=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \ + scripts/sign-file "${SIG_HASH}" "/linux/${SIG_KEY_SRCPREFIX}${SIG_KEY}" /linux/certs/signing_key.x509 $(find /tmp/kernel-modules/lib/modules -name 8821cu.ko); \ + fi + # Device Tree Blobs RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then make INSTALL_DTBS_PATH=/tmp/kernel-modules/boot/dtb CROSS_COMPILE="${CROSS_COMPILE_ENV}" ARCH="${KERNEL_ARCH}" dtbs_install ;fi diff --git a/pkg/kernel/certs/x509.genkey b/pkg/kernel/certs/x509.genkey new file mode 100644 index 0000000000..79910c3766 --- /dev/null +++ b/pkg/kernel/certs/x509.genkey @@ -0,0 +1,17 @@ +[ req ] +default_bits = 4096 +distinguished_name = req_distinguished_name +prompt = no +string_mask = utf8only +x509_extensions = myexts + +[ req_distinguished_name ] +O = LF Edge EVE-OS +CN = Build-time generated signing key +emailAddress = eve-security@lists.lfedge.org + +[ myexts ] +basicConstraints=critical,CA:FALSE +keyUsage=digitalSignature +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid diff --git a/pkg/kernel/kernel-config/kernel_config-5.10.x-aarch64 b/pkg/kernel/kernel-config/kernel_config-5.10.x-aarch64 index e56fc6e2e2..e9d3d74164 100644 --- a/pkg/kernel/kernel-config/kernel_config-5.10.x-aarch64 +++ b/pkg/kernel/kernel-config/kernel_config-5.10.x-aarch64 @@ -867,8 +867,8 @@ CONFIG_MODVERSIONS=y CONFIG_ASM_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG=y -# CONFIG_MODULE_SIG_FORCE is not set -# CONFIG_MODULE_SIG_ALL is not set +CONFIG_MODULE_SIG_FORCE=y +CONFIG_MODULE_SIG_ALL=y # CONFIG_MODULE_SIG_SHA1 is not set # CONFIG_MODULE_SIG_SHA224 is not set CONFIG_MODULE_SIG_SHA256=y @@ -6974,7 +6974,7 @@ CONFIG_PKCS7_MESSAGE_PARSER=y # # Certificates for signature checking # -CONFIG_MODULE_SIG_KEY="" +CONFIG_MODULE_SIG_KEY="certs/signing_key.pem" CONFIG_SYSTEM_TRUSTED_KEYRING=y CONFIG_SYSTEM_TRUSTED_KEYS="" # CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set diff --git a/pkg/kernel/kernel-config/kernel_config-5.10.x-x86_64 b/pkg/kernel/kernel-config/kernel_config-5.10.x-x86_64 index 1e065ce886..6f03dc801d 100644 --- a/pkg/kernel/kernel-config/kernel_config-5.10.x-x86_64 +++ b/pkg/kernel/kernel-config/kernel_config-5.10.x-x86_64 @@ -801,13 +801,22 @@ CONFIG_GCC_PLUGINS=y CONFIG_RT_MUTEXES=y CONFIG_BASE_SMALL=0 +CONFIG_MODULE_SIG_FORMAT=y CONFIG_MODULES=y # CONFIG_MODULE_FORCE_LOAD is not set CONFIG_MODULE_UNLOAD=y # CONFIG_MODULE_FORCE_UNLOAD is not set # CONFIG_MODVERSIONS is not set # CONFIG_MODULE_SRCVERSION_ALL is not set -# CONFIG_MODULE_SIG is not set +CONFIG_MODULE_SIG=y +CONFIG_MODULE_SIG_FORCE=y +CONFIG_MODULE_SIG_ALL=y +# CONFIG_MODULE_SIG_SHA1 is not set +# CONFIG_MODULE_SIG_SHA224 is not set +CONFIG_MODULE_SIG_SHA256=y +# CONFIG_MODULE_SIG_SHA384 is not set +# CONFIG_MODULE_SIG_SHA512 is not set +CONFIG_MODULE_SIG_HASH="sha256" # CONFIG_MODULE_COMPRESS is not set # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set # CONFIG_UNUSED_SYMBOLS is not set @@ -5599,6 +5608,7 @@ CONFIG_PKCS7_MESSAGE_PARSER=y # # Certificates for signature checking # +CONFIG_MODULE_SIG_KEY="certs/signing_key.pem" CONFIG_SYSTEM_TRUSTED_KEYRING=y CONFIG_SYSTEM_TRUSTED_KEYS="" # CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set diff --git a/tools/rootfs-diff.sh b/tools/rootfs-diff.sh new file mode 100755 index 0000000000..a69bba7556 --- /dev/null +++ b/tools/rootfs-diff.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +# +# Copyright (c) 2023 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# +# When module signing is enabled, the kernel builder will generate a temporary key +# to sing the kernel modules, this key is then discarded at the end of +# build process but the public part of the key is stored in the kernel key ring +# This results in having different kernels and modules hash each time kernel is +# built from the source. +# To make diff rootfs created from separate builds easier, this script will +# remove the public key form the kernel binary and strips the modules of the +# signature and then performs a diff on the given rootfs archives. +# +# before running the script, create necessary directories : +#mkdir /tmp/{rootfs-one,rfs-one-up,rfs-one-work,rfs-one-merged} +#mkdir /tmp/{rootfs-two,rfs-two-up,rfs-two-work,rfs-two-merged} +# +# mount the ro rootfs and a rw overlay on top of it: +#mount -o loop rootfs-one.img /tmp/rootfs-one +#mount -o loop rootfs-two.img /tmp/rootfs-two +#mount -t overlay overlay -o lowerdir=/tmp/rootfs-one,upperdir=/tmp/rfs-one-up,workdir=/tmp/rfs-one-work /tmp/rfs-one-merged +#mount -t overlay overlay -o lowerdir=/tmp/rootfs-two,upperdir=/tmp/rfs-two-up,workdir=/tmp/rfs-two-work /tmp/rfs-two-merged +# +# run the script: +#rootfs-diff.sh /tmp/rfs-one-merged/ /tmp/rfs-two-merged/ +# +# then clean up : +#umount /tmp/rfs-one-merged +#umount /tmp/rfs-two-merged +#umount /tmp/rootfs-one +#umount /tmp/rootfs-two +#rm -rf /tmp/{rootfs-one,rfs-one-up,rfs-one-work,rfs-one-merged} +#rm -rf /tmp/{rootfs-two,rfs-two-up,rfs-two-work,rfs-two-merged} + +if [ $# -lt 2 ] ; then + echo 'kdiff.sh [path-to-rootfs-one] [path-to-rootfs-two]' + exit 1 +fi + +STRIP=${STRIP:-strip} +for i in 1 2 +do + if [ $i -eq 1 ]; then ROOTFS=$1; else ROOTFS=$2; fi + MODULES="$(find "$ROOTFS"/lib/modules/ -mindepth 1 -maxdepth 1)" + KERNEL="$ROOTFS/boot/kernel" + X509="$MODULES/signing_key.x509" + GZK=0 + # aarch64 kernel + if (file "$KERNEL" | grep -q gzip ) ; then + cp "$KERNEL" /tmp/kernel.$i.gz || exit 1 + gunzip /tmp/kernel.$i.gz || exit 1 + KERNEL="/tmp/kernel.$i" + GZK=1 + fi + + IN_SIZE=$(stat -c %s "$X509") + IN=$(hexdump -ve '1/1 "%.2x"' "$X509") + OUT=$(head -c "$IN_SIZE" < /dev/zero | hexdump -ve '1/1 "%.2x"') + echo "[$i] Removing the signing key from kernel..." + hexdump -ve '1/1 "%.2X"' "$KERNEL" | sed "s/$IN/$OUT/I" | xxd -r -p > "$ROOTFS/boot/kernel" + echo "[$i] Removing the signature form kernel modules..." + # shellcheck disable=SC2046 + $STRIP --strip-debug $(find "$MODULES" -name \*.ko) + + if [ $GZK -eq 1 ]; then rm "$KERNEL";fi + # removing singing pub key that differs between builds + rm "$X509" + # removing files that differ because of timestamps + rm "$ROOTFS/etc/eve-release" + rm "$ROOTFS/etc/linuxkit-eve-config.yml" +done + +echo "[*] Diffing the two rootfs..." +diff -qr "$1" "$2" 2>/dev/null