Rework initrd usr mount to start Ignition/Afterburn from there

The size of the unified kernel image grew so much that we almost hit
the limit of the fixed /boot partition. While we could tweak which
kernel modules are present the largest contributors are actually
Ignition and Afterburn (coreos-metadata). Since we also ship Afterburn
on the /usr partition and have Ignition installed there, too (but
currently masked), we could do what was done for Torcx and directly
call it from there (but with LD_LIBRARY_PATH instead of chroot). There
are a few corner cases where this was not directly possible due to the
order of events in the initrd and these had to be reworked.

This establishes a structure that allows the /sysusr/usr mount to be
used for calling Ignition and Afterburn. This mount point was set up
by systemd already but since we have a legacy generator, this needed
adaption, too. The final /usr mount in /sysroot/usr is still set up.
In the case of the systemd generator it is a bind mount, for the
legacy generator it is a separate mount, both work but we could align
this to also use a bind mount. The afterburn-network-kargs.service
starts a bit later now because it depends on the /sysusr mount and
this also means that the parse-ip-for-networkd dracut hook needed to
run at a later stage, here solved with an own service but since the
dracut hook mechanism with sourcing is complicated, the script still
runs as hook to serialize the environment. The fsck check for /usr was
also in the way and thus we write out a dummy systemd-fsck-usr.service
since we don't need to check /usr as it's verity-protected. In the end
with these changes we significantly reduce the unified kernel image
size and have more breathing room for the next years. If we have to
move some other binaries or even kernel modules that are not essential
for loading /usr itself, we can move them out of the initrd based on
this work. Size reduction for arm64 is 58 MB -> 52 MB, for amd64 it is
55 MB -> 49MB.

dracut/03flatcar-network/afterburn-network-kargs.service

@@ -5,14 +5,18 @@ Documentation=https://coreos.github.io/afterburn/usage/initrd-network-cmdline/
 # This service may produce additional kargs fragments,
 # which are then consumed by dracut-cmdline(8).
 DefaultDependencies=no
-Before=dracut-cmdline.service systemd-networkd.service
+Before=parse-ip-for-networkd.service systemd-networkd.service
 PartOf=systemd-networkd.service
 # For extra safety
 ConditionKernelCommandLine=|coreos.oem.id=vmware
 ConditionKernelCommandLine=|flatcar.oem.id=vmware
 OnFailure=emergency.target
 OnFailureJobMode=isolate
 
+# Flatcar: Load coreos-metadata binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
+
 [Service]
 ExecStart=/usr/bin/coreos-metadata exp rd-network-kargs --cmdline --default-value ''
 Type=oneshot

dracut/03flatcar-network/module-setup.sh

@@ -10,8 +10,6 @@ depends() {
 
 # called by dracut
 install() {
-    inst_multiple coreos-metadata
-
     inst_multiple -o \
         $systemdutildir/systemd-resolved \
         $systemdsystemunitdir/systemd-resolved.service \
@@ -20,6 +18,9 @@ install() {
     inst_simple "$moddir/network-cleanup.service" \
         "$systemdsystemunitdir/network-cleanup.service"
 
+    inst_simple "$moddir/parse-ip-for-networkd.service" \
+        "$systemdsystemunitdir/parse-ip-for-networkd.service"
+
     inst_simple "$moddir/afterburn-network-kargs.service" \
         "$systemdsystemunitdir/afterburn-network-kargs.service"
 
@@ -61,5 +62,6 @@ install() {
     systemctl --root "$initdir" disable systemd-networkd.socket
 
     systemctl --root "$initdir" enable network-cleanup.service
+    systemctl --root "$initdir" enable parse-ip-for-networkd.service
     systemctl --root "$initdir" enable afterburn-network-kargs.service
 }

dracut/03flatcar-network/parse-ip-for-networkd.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Write systemd-networkd units from cmdline
+DefaultDependencies=false
+
+After=afterburn-network-kargs.service
+PartOf=systemd-networkd.service
+Before=systemd-networkd.service initrd-switch-root.target
+# Switching the root filesystem terminates all running services with binaries from the initramfs, we need to finish before that happens
+Conflicts=initrd-switch-root.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+Environment="APPLY=1"
+ExecStart=/lib/dracut/hooks/cmdline/99-parse-ip-for-networkd.sh
+
+[Install]
+WantedBy=systemd-networkd.service

dracut/03flatcar-network/parse-ip-for-networkd.sh

@@ -1,17 +1,59 @@
-#!/bin/sh
+#!/bin/bash
 #
 # This script was mostly stolen from 40network/parse-ip-opts.sh.  Its
 # actions are adapted to write .network files to /etc/systemd/network
 # in the initramfs instead of using separate DHCP commands, etc.  Note
-# the bashisms.
+# the bashisms. It used to be run as a dracut hook, relying on afterburn to
+# process kargs for the dracut cmdline target. Now that we start afterburn
+# from /sysusr instead of ramdisk root, we need to mount /sysusr first. Mounting
+# happens later than the dracut hook we used. Thus we added a systemd unit for
+# this script. The unit runs shortly before systemd-networkd.
+# However, since the "netroot" variable is set up by another dracut
+# hook, this script here will still get executed once as dracut hook
+# to save the "netroot" variable to an env file and exit.
+# Could the script instead be a hook at a later stage? There doesn't
+# seem to be a suitable later stage and also then the env could be
+# different. As long as we want to rely on dracut-lib/net-lib for
+# parsing the env var topic is not easily avoided.
 #
 
+if [ "${APPLY-}" != "1" ]; then
+  # First run, called as dracut hook
+  {
+  # While the script only makes use of the env vars "NEEDBOOTDEV" and "netroot"
+  # we actually don't really know what env vars dracut-lib.sh and net-lib.sh
+  # will depend on in future versions and therefore, we try to preserve the
+  # environment as is.
+  for VARNAME in $(compgen -v); do
+    # Prevent leaking HEREEOF into VAL ($_ is the argument of the prev. command, and in this loop contains HEREOF)
+    [ "${VARNAME}" = "_" ] && continue
+    # Skip unnecessary variables
+    [[ "${VARNAME}" = "BASH"* ]] && continue
+    # Skip errors from read-only variables
+    (unset "${VARNAME}" 2> /dev/null) || continue
+    VAL="${!VARNAME}"
+    echo "${VARNAME}=\$(cat <<'HEREEOF'
+${VAL}
+HEREEOF
+)"
+  done
+  } > /saved-parse-ip.env
+  return;
+else
+  # Second run, expected to be called as systemd unit
+  # Make it a hard error if we forgot to exclude some problematic variables and thus the sourcing terminates without setting all variables
+  . /saved-parse-ip.env || { echo "Error: failed sourcing all variables"; exit 1 ; }
+fi
+
+# The getarg uses getcmdline which assembles the cmdline on-the-fly
+# from /proc/cmdline and the drop-in files under /etc/cmdline.d/
+# where afterburn could have written the kargs values
 command -v getarg >/dev/null          || . /lib/dracut-lib.sh
 command -v ip_to_var >/dev/null       || . /lib/net-lib.sh
 
 if [ -n "$netroot" ] && [ -z "$(getarg ip=)" ] && [ -z "$(getarg BOOTIF=)" ]; then
     # No ip= argument(s) for netroot provided, defaulting to DHCP
-    return;
+    exit 0
 fi
 
 function mask2cidr() {
@@ -30,6 +72,9 @@ function mask2cidr() {
 # XXX Would be nice if we could errorcheck ip addresses here as well
 for p in $(getargs ip=); do
     ip_to_var $p
+    # From here on the variables "ip", "mask" etc are set up (or are cleared)
+    # from 'unset ip srv gw mask hostname dev autoconf macaddr mtu dns1 dns2' in ip_to_var
+    # ("cidr" is defined below)
 
     # Empty autoconf defaults to 'dhcp'
     if [ -z "$autoconf" ] ; then
@@ -89,3 +134,5 @@ for p in $(getargs ip=); do
     echo '[DHCP]' >> $_net_file
     [ -n "$hostname" ] && echo "Hostname=$hostname" >> $_net_file
 done
+# Have a clear exit code instead of propagating the one from [ -n "$hostname" ]
+exit 0

dracut/10diskless-generator/diskless-generator

@@ -52,6 +52,22 @@ After=systemd-tmpfiles-setup-dev.service
 What=/usr.squashfs
 Where=/sysroot/usr
 Type=squashfs
+EOF
+    mkdir -p /sysusr/usr
+    cat >"${UNIT_DIR}/sysusr-usr.mount" <<EOF
+# Automatically generated by diskless-generator
+[Unit]
+# Make sure the loop device nodes are available
+Wants=systemd-tmpfiles-setup-dev.service
+After=systemd-tmpfiles-setup-dev.service
+Conflicts=initrd-switch-root.target
+DefaultDependencies=no
+After=systemd-udevd.service
+
+[Mount]
+What=/usr.squashfs
+Where=/sysusr/usr
+Type=squashfs
 EOF
 fi
 

dracut/10usr-generator/usr-generator

@@ -32,6 +32,22 @@ cmdline_arg() {
     echo "${value}"
 }
 
+# Overwrite systemd-fsck-usr.service because it wants After=local-fs-pre.target
+# which prevents (the systemd-generated) sysusr-usr.mount from starting before Ignition
+# but we need it because we want to start Ignition from /sysusr/usr/ already.
+# Use /run/systemd/system/ because the generator folder already is the location
+# of the original systemd-fsck-usr.service and generators may have the wrong order.
+mkdir -p /run/systemd/system/
+cat > /run/systemd/system/systemd-fsck-usr.service <<EOF
+# Automatically generated by usr-generator
+[Unit]
+DefaultDependencies=no
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=true
+EOF
+
 usr=$(cmdline_arg usr)
 usrfstype=$(cmdline_arg usrfstype auto)
 usrflags=$(cmdline_arg usrflags ro)
@@ -78,6 +94,10 @@ if [[ "${usr}" != /* ]]; then
   # sysroot-usr.mount is a bind mount from /sysusr/usr
   mkdir -p "${UNIT_DIR}/sysusr-usr.mount.d"
   cat >"${UNIT_DIR}/sysusr-usr.mount.d/10-norecovery.conf" <<EOF
+[Unit]
+DefaultDependencies=no
+Wants=systemd-tmpfiles-setup-dev.service
+After=systemd-udevd.service systemd-tmpfiles-setup-dev.service
 [Mount]
 Options=${mount_usrflags}
 EOF
@@ -103,3 +123,21 @@ EOF
 requires_dir="${UNIT_DIR}/initrd-root-fs.target.requires"
 mkdir -p "${requires_dir}"
 ln -sf "../sysroot-usr.mount" "${requires_dir}/sysroot-usr.mount"
+
+mkdir -p /sysusr/usr
+cat >"${UNIT_DIR}/sysusr-usr.mount" <<EOF
+# Automatically generated by usr-generator
+
+[Unit]
+SourcePath=/proc/cmdline
+Conflicts=initrd-switch-root.target
+DefaultDependencies=no
+Wants=systemd-tmpfiles-setup-dev.service
+After=systemd-udevd.service systemd-tmpfiles-setup-dev.service
+
+[Mount]
+What=${usr}
+Where=/sysusr/usr
+Type=${usrfstype}
+Options=${usrflags}
+EOF

dracut/10verity-generator/verity-generator

@@ -69,7 +69,7 @@ if [[ -n "${usr}" && -n "${usrhash}" ]]; then
 	IgnoreOnIsolate=true
 	BindsTo=dev-mapper-usr.device
 	BindsTo=${device}
-	After=${device} ignition-disks.service
+	After=${device}
 
 	[Service]
 	Type=oneshot

dracut/30ignition/coreos-metadata-wrapper

@@ -0,0 +1,2 @@
+#!/bin/sh
+LD_LIBRARY_PATH=/sysusr/usr/lib64 exec /sysusr/usr/bin/coreos-metadata "$@"

dracut/30ignition/flatcar-digitalocean-network.service

@@ -4,6 +4,9 @@ DefaultDependencies=false
 Before=initrd.target
 After=systemd-networkd.service initrd-root-fs.target
 Wants=systemd-networkd.service initrd-root-fs.target
+# Flatcar: Load coreos-metadata binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
 
 [Service]
 Type=oneshot

dracut/30ignition/flatcar-metadata-hostname.service

@@ -4,6 +4,9 @@ DefaultDependencies=false
 Before=initrd.target
 After=systemd-networkd.service initrd-root-fs.target
 Wants=systemd-networkd.service initrd-root-fs.target
+# Flatcar: Load coreos-metadata binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
 
 # Ensure Ignition can overwrite /etc/hostname
 Before=ignition-files.service

dracut/30ignition/flatcar-openstack-hostname.service

@@ -4,6 +4,9 @@ DefaultDependencies=false
 Before=initrd.target
 After=systemd-networkd.service initrd-root-fs.target
 Wants=systemd-networkd.service initrd-root-fs.target
+# Flatcar: Load coreos-metadata binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
 
 # Ensure Ignition can overwrite /etc/hostname
 Before=ignition-files.service

dracut/30ignition/flatcar-static-network.service

@@ -4,6 +4,9 @@ DefaultDependencies=false
 Before=initrd.target
 After=systemd-networkd.service initrd-root-fs.target
 Wants=systemd-networkd.service initrd-root-fs.target
+# Flatcar: Load coreos-metadata binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
 
 # Ensure Ignition can overwrite /etc/hostname
 Before=ignition-files.service

dracut/30ignition/ignition-generator

@@ -109,6 +109,10 @@ cat > ${UNIT_DIR}/ignition-setup.service <<EOF
 Description=Ignition (setup)
 DefaultDependencies=false
 
+# Flatcar: Load Ignition binary
+Requires=sysusr-usr.mount
+After=sysusr-usr.mount
+
 Requires=local-fs-pre.target
 Before=local-fs-pre.target
 

dracut/30ignition/ignition-wrapper

@@ -0,0 +1,2 @@
+#!/bin/sh
+LD_LIBRARY_PATH=/sysusr/usr/lib64 exec /sysusr/usr/bin/ignition "$@"

dracut/30ignition/module-setup.sh

@@ -19,7 +19,6 @@ install_ignition_unit() {
 install() {
     # Flatcar: add coreos-metadata, systemd-detect-virt, mountpoint, nvme
     inst_multiple \
-        coreos-metadata \
         basename \
         lsblk
 
@@ -74,9 +73,6 @@ install() {
     inst_script "$moddir/ignition-setup.sh" \
         "/usr/sbin/ignition-setup"
 
-    # Flatcar: use from path
-    inst_multiple "ignition"
-
     # Rule to allow udev to discover unformatted encrypted devices
     inst_simple "$moddir/99-xx-ignition-systemd-cryptsetup.rules" \
         "/usr/lib/udev/rules.d/99-xx-ignition-systemd-cryptsetup.rules"
@@ -127,4 +123,11 @@ install() {
     # needed for openstack config drive support
     # Flatcar: add 66-azure-storage.rules and 90-cloud-storage.rules
     inst_rules 60-cdrom_id.rules 66-azure-storage.rules 90-cloud-storage.rules
+
+    # Flatcar: add wrappers for Ignition and coreos-metadata (afterburn)
+    # which rely on the earlier /sysusr/usr mount
+    inst_script "$moddir/coreos-metadata-wrapper" \
+        "/usr/bin/coreos-metadata"
+    inst_script "$moddir/ignition-wrapper" \
+        "/usr/bin/ignition"
 }