05core: add coreos-ignition-delete-config.service for upgrade boots

Ignition 2.14.0 adds ignition-delete-config.service, which deletes
Ignition configs from VMware and VirtualBox on first boot.  Add
coreos-ignition-delete-config.service to do the same thing on existing
machines on upgrade, using a stamp file in /var/lib to avoid multiple
runs.

Add a drop-in for ignition-delete-config.service that creates a stamp
file in /run, and then chain from that stamp to the long-term stamp in
/var/lib, ensuring that we don't delete configs twice on newly-
provisioned machines.  The upstream service can't create the stamp
directly in /var/lib because it runs before /var is mounted.

Prevent coreos-ignition-delete-config.service from running if
ignition-delete-config.service is masked, ensuring that the mask operation
documented upstream prevents the config from ever being deleted, as
intended.

We can remove this after the next barrier release in FCOS and barrier
equivalent in RHCOS.

overlay.d/05core/usr/lib/systemd/system-preset/40-coreos.preset

@@ -10,6 +10,8 @@ enable coreos-ignition-firstboot-complete.service
 # Delete Ignition config from provider on platforms where it's possible
 # https://github.com/coreos/ignition/pull/1350
 enable ignition-delete-config.service
+# Delete Ignition config from provider when upgrading existing nodes
+enable coreos-ignition-delete-config.service
 # Boot checkin services for cloud providers.
 enable afterburn-checkin.service
 enable afterburn-firstboot-checkin.service

overlay.d/05core/usr/lib/systemd/system/coreos-ignition-delete-config.service

@@ -0,0 +1,28 @@
+# Can be removed from FCOS in Fedora 37 or after the next barrier release,
+# whichever comes first.  Can be removed from RHCOS in the first release
+# after every node is guaranteed to have booted at least once with 4.11 or
+# higher.
+
+[Unit]
+Description=CoreOS Delete Ignition Config From Hypervisor
+Documentation=https://coreos.github.io/ignition/
+
+ConditionKernelCommandLine=|ignition.platform.id=virtualbox
+ConditionKernelCommandLine=|ignition.platform.id=vmware
+ConditionPathExists=!/var/lib/coreos-ignition-delete-config.stamp
+# Hack: if the user masked ignition-delete-config.service, we shouldn't run
+# either.
+ConditionPathIsSymbolicLink=!/etc/systemd/system/ignition-delete-config.service
+
+# We check a stamp file written by ignition-delete-config.service.  That
+# service runs Before=sysinit.target, on which we have a default dependency,
+# so this is really just documentation.
+After=ignition-delete-config.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-ignition-delete-config
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

overlay.d/05core/usr/lib/systemd/system/ignition-delete-config.service.d/10-flag-file.conf

@@ -0,0 +1,7 @@
+# Create a flag file to notify coreos-ignition-delete-config.service that
+# we've run, and put it in /run because /var isn't mounted yet.
+# coreos-ignition-delete-config.service will then avoid trying to delete
+# the config again, and will create a persistent stamp file in /var/lib.
+
+[Service]
+ExecStart=/bin/touch /run/coreos-ignition-delete-config.stamp

overlay.d/05core/usr/libexec/coreos-ignition-delete-config

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cmdline=( $(</proc/cmdline) )
+cmdline_arg() {
+    local name="$1" value
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+
+# Avoid running again if ignition-delete-config.service has run, but still
+# create our own stamp file now that /var is mounted.
+if [ ! -e /run/coreos-ignition-delete-config.stamp ]; then
+    PLATFORM_ID=$(cmdline_arg ignition.platform.id)
+    /usr/libexec/ignition-rmcfg --platform=${PLATFORM_ID}
+fi
+
+touch /var/lib/coreos-ignition-delete-config.stamp

tests/kola/ignition/delete-config/config.bu

@@ -0,0 +1,36 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/fake-ignition-rmcfg
+      mode: 0755
+      contents:
+        inline: |
+          #!/bin/bash
+          # Mocked replacement for ignition-rmcfg that just records that
+          # we ran it, and fails if we run it twice.  We can't run
+          # ignition-rmcfg directly because it doesn't succeed on any
+          # platform we test on.
+          if [ -e /run/ignition-rmcfg-ran ]; then
+              echo "ignition-rmcfg ran twice"
+              exit 1
+          fi
+          touch /run/ignition-rmcfg-ran
+systemd:
+  units:
+    - name: ignition-delete-config.service
+      dropins:
+        - name: 50-kola.conf
+          contents: |
+            [Unit]
+            ConditionKernelCommandLine=|ignition.platform.id=qemu
+            [Service]
+            ExecStartPre=mount --bind /etc/fake-ignition-rmcfg /usr/libexec/ignition-rmcfg
+    - name: coreos-ignition-delete-config.service
+      dropins:
+        - name: 50-kola.conf
+          contents: |
+            [Unit]
+            ConditionKernelCommandLine=|ignition.platform.id=qemu
+            [Service]
+            ExecStartPre=mount --bind /etc/fake-ignition-rmcfg /usr/libexec/ignition-rmcfg

tests/kola/ignition/delete-config/data/commonlib.sh

@@ -0,0 +1 @@
+../../../data/commonlib.sh

tests/kola/ignition/delete-config/test.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# kola: { "platforms": "qemu-unpriv" }
+# - platforms: qemu-unpriv
+#   - Ideally we'd test on virtualbox and vmware, but we don't have tests
+#     there, so we mock specifically for ignition.platform.id=qemu
+
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+"")
+    # Ignition boot
+
+    if [ ! -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg did not run on first boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != active ]; then
+        fatal "ignition-delete-config didn't succeed on first boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != active ]; then
+        fatal "coreos-ignition-delete-config didn't succeed on first boot"
+    fi
+    ok "First boot OK"
+
+    # Reset state and reboot
+    rm /var/lib/coreos-ignition-delete-config.stamp
+    /tmp/autopkgtest-reboot upgrade
+    ;;
+
+upgrade)
+    # Simulated upgrade from Ignition < 2.14.0
+
+    if [ ! -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg did not run on upgrade boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on upgrade boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != active ]; then
+        fatal "coreos-ignition-delete-config didn't succeed on upgrade boot"
+    fi
+    ok "Upgrade boot OK"
+
+    /tmp/autopkgtest-reboot steady-state
+    ;;
+
+steady-state)
+    # Steady-state boot; nothing should run
+
+    if [ -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg ran on steady-state boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on steady-state boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != inactive ]; then
+        fatal "coreos-ignition-delete-config ran on steady-state boot"
+    fi
+    ok "Steady-state boot OK"
+
+    # Reset state for masked unit and reboot
+    rm /var/lib/coreos-ignition-delete-config.stamp
+    systemctl mask ignition-delete-config.service
+    /tmp/autopkgtest-reboot masked
+    ;;
+
+masked)
+    # Simulated upgrade with masked ignition-delete-config.service
+
+    if [ -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg ran on masked boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on masked boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != inactive ]; then
+        fatal "coreos-ignition-delete-config ran on masked boot"
+    fi
+    ok "Masked unit OK"
+    ;;
+
+*) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}" ;;
+esac