lf-edge · eriknordmark · May 23, 2023 · Mar 23, 2023
@@ -15,3 +15,4 @@ yetus-output/
 assets/
 pkg/kernel/build.yml
 pkg/new-kernel/build.yml
+pkg/kernel/certs/*.pem
@@ -506,7 +506,7 @@ docker run -p 8080:80 -p 8125:8125/udp --rm --name statsd graphiteapp/graphite-s
 
 While the EVE is running, navigate to `http://IP:8080/dashboard` and find the result under `stats.gauges`.
 
-#### Kernel versioning
+#### Reproducible Kernel build and versioning
 
 Kernel packages ("pkg/kernel" and "pkg/new-kernel") are configured to produce a bit-by-bit reproducible kernel, to this end a `build.yml` is generated at build time to set the following variables to a static value:
 
@@ -518,7 +518,43 @@ KCONFIG_NOTIMESTAMP=true
 
 In addition, both `KBUILD_BUILD_TIMESTAMP` and `SOURCE_DATE_EPOCH` variables are set to the last commit date of the respective package, This configuration results in having a static version string (`/proc/version`) on every build. In case there is uncommitted changes in the kernel(s) directory, kernel gets build normally without any static time.
 
-This process can be used to compare eve images that are build in a trusted environment vs CI, making sure the automated build process is intact and not malicious or compromised.
+This process can be used to compare eve images that are build in a trusted environment vs CI, making sure the automated build process is intact and not malicious or compromised. For this purpose, you can use [rootfs-diff.sh](../tools/rootfs-diff.sh) to compare two builds. The script accepts the path of the mounted rootfs.img files (you need to mount a RW overlay on top; check the comments on the script), ideally you should see no output after the diff is finished:
+
+```bash
+$ rootfs-diff.sh /tmp/rfs-one/ tmp/rfs-two/
+[1] Removing the signing key from kernel...
+[1] Removing the signature form kernel modules...
+[2] Removing the signing key from kernel...
+[2] Removing the signature form kernel modules...
+[*] Diffing the two rootfs...
+$
+```
+
+But in case the two builds differ, the script outputs a list of files:
+
+```bash
+$ rootfs-diff.sh /tmp/rfs-one/ tmp/rfs-two/
+[1] Removing the signing key from kernel...
+[1] Removing the signature form kernel modules...
+[2] Removing the signing key from kernel...
+[2] Removing the signature form kernel modules...
+[*] Diffing the two rootfs...
+Files /tmp/rfs-one/boot/kernel and /tmp/rfs-two/boot/kernel differ
+Files /tmp/rfs-one/lib/modules/5.10.121-default/kernel/net/can/can.ko and /tmp/rfs-two/lib/modules/5.10.121-default/kernel/net/can/can.ko differ
+$
+```
+
+#### Buil-time kernel module signing
+
+EVE kernel is configured to only load signed kernel modules, the module signing happens automatically during the build with a build-time generated throw-away key. But if you are willing to sign the modules using your own key, generate a key:
+
+```shell
+openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \
+   -config x509.genkey -outform PEM -out kernel_key.pem \
+   -keyout kernel_key.pem
+```
+
+and place the `.pem` file in the `/eve/pkg/kernel/certs` directory. You can respectively change the `x509.genkey` template too.
 
 ## Summary of Build Process
 

@@ -79,6 +79,7 @@ ENV KERNEL_DEFCONFIG=x86_64_defconfig
 FROM kernel-target-${TARGETARCH} AS kernel-build
 
 COPY /kernel-config/* /
+COPY /certs /certs
 COPY /patches-5.10.x /patches-5.10.x
 COPY /patches-zfs-2.1.2 /patches-zfs-2.1.2
 
@@ -113,6 +114,15 @@ RUN set -e ; KERNEL_SERIES="${KERNEL_VERSION%.*}".x; \
         patch -p1 < "$patch"; \
     done
 
+# Copy module signing x509 template and key
+WORKDIR /linux
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+RUN cp /certs/x509.genkey /linux/certs/; \
+    if find certs/ -name \*.pem | grep . ; then \
+        KEY=$(find certs/ -name \*.pem | head -1); \
+        cp "${KEY}" /linux/certs/signing_key.pem; \
+    fi
+
 # Copy default kconfig and prepare kbuild
 # hadolint ignore=SC2086
 RUN KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \
@@ -216,6 +226,26 @@ RUN make -j "$(getconf _NPROCESSORS_ONLN)" CROSS_COMPILE="${CROSS_COMPILE_ENV}"
 # Strip at least some of the modules to conserve space
 RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then "${CROSS_COMPILE_ENV}strip" --strip-debug `find /tmp/kernel-modules/lib/modules -name \*.ko` ;fi
 
+# Resign stripped kernel modules again
+RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then \
+    KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \
+    if grep -q "^CONFIG_MODULE_SIG_FORCE=y" "${KERNEL_DEF_CONF}" ;then \
+        make INSTALL_MOD_PATH=/tmp/kernel-modules modules_sign ; \
+    fi \
+fi
+
+# Sign out-of-tree module(s)
+WORKDIR /linux
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+# hadolint ignore=SC2086,SC2046
+RUN KERNEL_DEF_CONF="/linux/arch/${KERNEL_ARCH}/configs/${KERNEL_DEFCONFIG}"; \
+    if grep -q "^CONFIG_MODULE_SIG_FORCE=y" "${KERNEL_DEF_CONF}" ;then \
+        SIG_HASH=$(sed -n '/^CONFIG_MODULE_SIG_HASH=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \
+        SIG_KEY_SRCPREFIX=$(sed -n '/^MODULE_SIG_KEY_SRCPREFIX=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \
+        SIG_KEY=$(sed -n '/^CONFIG_MODULE_SIG_KEY=/s///p' ${KERNEL_DEF_CONF} | tr -d '"') ; \
+        scripts/sign-file "${SIG_HASH}" "/linux/${SIG_KEY_SRCPREFIX}${SIG_KEY}" /linux/certs/signing_key.x509 $(find /tmp/kernel-modules/lib/modules -name 8821cu.ko); \
+    fi
+
 # Device Tree Blobs
 RUN if [ "${EVE_TARGET_ARCH}" = aarch64 ];then make INSTALL_DTBS_PATH=/tmp/kernel-modules/boot/dtb CROSS_COMPILE="${CROSS_COMPILE_ENV}" ARCH="${KERNEL_ARCH}" dtbs_install ;fi
 

@@ -0,0 +1,17 @@
+[ req ]
+default_bits = 4096
+distinguished_name = req_distinguished_name
+prompt = no
+string_mask = utf8only
+x509_extensions = myexts
+
+[ req_distinguished_name ]
+O = LF Edge EVE-OS
+CN = Build-time generated signing key
+emailAddress = eve-security@lists.lfedge.org
+
+[ myexts ]
+basicConstraints=critical,CA:FALSE
+keyUsage=digitalSignature
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid
@@ -867,8 +867,8 @@ CONFIG_MODVERSIONS=y
 CONFIG_ASM_MODVERSIONS=y
 CONFIG_MODULE_SRCVERSION_ALL=y
 CONFIG_MODULE_SIG=y
-# CONFIG_MODULE_SIG_FORCE is not set
-# CONFIG_MODULE_SIG_ALL is not set
+CONFIG_MODULE_SIG_FORCE=y
+CONFIG_MODULE_SIG_ALL=y
 # CONFIG_MODULE_SIG_SHA1 is not set
 # CONFIG_MODULE_SIG_SHA224 is not set
 CONFIG_MODULE_SIG_SHA256=y
@@ -6974,7 +6974,7 @@ CONFIG_PKCS7_MESSAGE_PARSER=y
 #
 # Certificates for signature checking
 #
-CONFIG_MODULE_SIG_KEY=""
+CONFIG_MODULE_SIG_KEY="certs/signing_key.pem"
 CONFIG_SYSTEM_TRUSTED_KEYRING=y
 CONFIG_SYSTEM_TRUSTED_KEYS=""
 # CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set

@@ -801,13 +801,22 @@ CONFIG_GCC_PLUGINS=y
 
 CONFIG_RT_MUTEXES=y
 CONFIG_BASE_SMALL=0
+CONFIG_MODULE_SIG_FORMAT=y
 CONFIG_MODULES=y
 # CONFIG_MODULE_FORCE_LOAD is not set
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_MODULE_FORCE_UNLOAD is not set
 # CONFIG_MODVERSIONS is not set
 # CONFIG_MODULE_SRCVERSION_ALL is not set
-# CONFIG_MODULE_SIG is not set
+CONFIG_MODULE_SIG=y
+CONFIG_MODULE_SIG_FORCE=y
+CONFIG_MODULE_SIG_ALL=y
+# CONFIG_MODULE_SIG_SHA1 is not set
+# CONFIG_MODULE_SIG_SHA224 is not set
+CONFIG_MODULE_SIG_SHA256=y
+# CONFIG_MODULE_SIG_SHA384 is not set
+# CONFIG_MODULE_SIG_SHA512 is not set
+CONFIG_MODULE_SIG_HASH="sha256"
 # CONFIG_MODULE_COMPRESS is not set
 # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
 # CONFIG_UNUSED_SYMBOLS is not set
@@ -5599,6 +5608,7 @@ CONFIG_PKCS7_MESSAGE_PARSER=y
 #
 # Certificates for signature checking
 #
+CONFIG_MODULE_SIG_KEY="certs/signing_key.pem"
 CONFIG_SYSTEM_TRUSTED_KEYRING=y
 CONFIG_SYSTEM_TRUSTED_KEYS=""
 # CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set

@@ -0,0 +1,77 @@
+#!/bin/sh
+
+#
+# Copyright (c) 2023 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# When module signing is enabled, the kernel builder will generate a temporary key
+# to sing the kernel modules, this key is then discarded at the end of
+# build process but the public part of the key is stored in the kernel key ring
+# This results in having different kernels and modules hash each time kernel is
+# built from the source.
+# To make diff rootfs created from separate builds easier, this script will
+# remove the public key form the kernel binary and strips the modules of the
+# signature and then performs a diff on the given rootfs archives.
+#
+# before running the script, create necessary directories :
+#mkdir /tmp/{rootfs-one,rfs-one-up,rfs-one-work,rfs-one-merged}
+#mkdir /tmp/{rootfs-two,rfs-two-up,rfs-two-work,rfs-two-merged}
+#
+# mount the ro rootfs and a rw overlay on top of it:
+#mount -o loop rootfs-one.img /tmp/rootfs-one
+#mount -o loop rootfs-two.img /tmp/rootfs-two
+#mount -t overlay overlay -o lowerdir=/tmp/rootfs-one,upperdir=/tmp/rfs-one-up,workdir=/tmp/rfs-one-work /tmp/rfs-one-merged
+#mount -t overlay overlay -o lowerdir=/tmp/rootfs-two,upperdir=/tmp/rfs-two-up,workdir=/tmp/rfs-two-work /tmp/rfs-two-merged
+#
+# run the script:
+#rootfs-diff.sh /tmp/rfs-one-merged/ /tmp/rfs-two-merged/
+#
+# then clean up :
+#umount /tmp/rfs-one-merged
+#umount /tmp/rfs-two-merged
+#umount /tmp/rootfs-one
+#umount /tmp/rootfs-two
+#rm -rf /tmp/{rootfs-one,rfs-one-up,rfs-one-work,rfs-one-merged}
+#rm -rf /tmp/{rootfs-two,rfs-two-up,rfs-two-work,rfs-two-merged}
+
+if [ $# -lt 2 ] ; then
+    echo 'kdiff.sh [path-to-rootfs-one] [path-to-rootfs-two]'
+    exit 1
+fi
+
+STRIP=${STRIP:-strip}
+for i in 1 2
+do
+    if [ $i -eq 1 ]; then ROOTFS=$1; else  ROOTFS=$2; fi
+    MODULES="$(find "$ROOTFS"/lib/modules/ -mindepth 1 -maxdepth 1)"
+    KERNEL="$ROOTFS/boot/kernel"
+    X509="$MODULES/signing_key.x509"
+    GZK=0
+    # aarch64 kernel
+    if (file "$KERNEL" | grep -q gzip ) ; then
+        cp "$KERNEL" /tmp/kernel.$i.gz || exit 1
+        gunzip /tmp/kernel.$i.gz || exit 1
+        KERNEL="/tmp/kernel.$i"
+        GZK=1
+    fi
+
+    IN_SIZE=$(stat -c %s "$X509")
+    IN=$(hexdump -ve '1/1 "%.2x"' "$X509")
+    OUT=$(head -c "$IN_SIZE" < /dev/zero | hexdump -ve '1/1 "%.2x"')
+    echo "[$i] Removing the signing key from kernel..."
+    hexdump -ve '1/1 "%.2X"' "$KERNEL" | sed "s/$IN/$OUT/I" | xxd -r -p > "$ROOTFS/boot/kernel"
+    echo "[$i] Removing the signature form kernel modules..."
+    # shellcheck disable=SC2046
+    $STRIP --strip-debug $(find "$MODULES" -name \*.ko)
+
+    if [ $GZK -eq 1 ]; then rm "$KERNEL";fi
+    # removing singing pub key that differs between builds
+    rm "$X509"
+    # removing files that differ because of timestamps
+    rm "$ROOTFS/etc/eve-release"
+    rm "$ROOTFS/etc/linuxkit-eve-config.yml"
+done
+
+echo "[*] Diffing the two rootfs..."
+diff -qr "$1" "$2" 2>/dev/null