From cfa0ed1625e08cdc6c531707224975368d19daaa Mon Sep 17 00:00:00 2001
From: CoreOS Bot <coreosbot@fedoraproject.org>
Date: Mon, 22 Apr 2024 13:37:29 +0000
Subject: [PATCH] tree: promote changes from testing-devel at
 0566985301cc7d1c330d02d358099e27e1ccb724

---
 .github/workflows/add-override.yml            |   2 +-
 .github/workflows/openshift-os.yml            |   2 +-
 .../workflows/remove-graduated-overrides.yml  |   2 +-
 ci/find-whitespace                            |  54 +++-
 image-base.yaml                               |   9 -
 kola-denylist.yaml                            |  19 +-
 manifest-lock.aarch64.json                    | 108 +++----
 manifest-lock.overrides.yaml                  |  14 +-
 manifest-lock.ppc64le.json                    | 108 +++----
 manifest-lock.s390x.json                      | 108 +++----
 manifest-lock.x86_64.json                     | 106 +++---
 manifests/fedora-coreos.yaml                  |   4 +-
 manifests/kdump.yaml                          |   8 +
 manifests/makedumpfile.yaml                   |   4 -
 .../coreos-diskful-generator                  |   3 +-
 .../modules.d/40ignition-conf/README.md       |   2 +-
 .../ignition-ostree-growfs.service            |   2 +-
 .../coreos-sulogin-force-generator            |   2 +-
 tests/kola/containers/quadlet/test.sh         |  27 +-
 .../kubernetes/systemd-env-read/config.bu     |   2 +-
 tests/kola/var-mount/scsi-id/config.bu        |   5 +-
 tests/kola/var-mount/scsi-id/test.sh          |   2 +-
 tests/manual/coreos-builds-bisect.py          | 303 ++++++++++++++++++
 23 files changed, 613 insertions(+), 283 deletions(-)
 create mode 100644 manifests/kdump.yaml
 delete mode 100644 manifests/makedumpfile.yaml
 create mode 100755 tests/manual/coreos-builds-bisect.py

diff --git a/.github/workflows/add-override.yml b/.github/workflows/add-override.yml
index 219e711f82..e243037f2f 100644
--- a/.github/workflows/add-override.yml
+++ b/.github/workflows/add-override.yml
@@ -80,7 +80,7 @@ jobs:
           echo "COMMIT_TITLE=${title}" >> ${GITHUB_ENV}
           echo "PR_TITLE=${pr_title}" >> ${GITHUB_ENV}
       - name: Open pull request
-        uses: peter-evans/create-pull-request@v4.2.3
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
           branch: ${{ env.BRANCH_NAME }}
diff --git a/.github/workflows/openshift-os.yml b/.github/workflows/openshift-os.yml
index b57f5943e3..90b1676b20 100644
--- a/.github/workflows/openshift-os.yml
+++ b/.github/workflows/openshift-os.yml
@@ -106,7 +106,7 @@ jobs:
           EOF
 
       - name: Open pull request
-        uses: peter-evans/create-pull-request@v6.0.0
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
           push-to-fork: coreosbot-releng/os
diff --git a/.github/workflows/remove-graduated-overrides.yml b/.github/workflows/remove-graduated-overrides.yml
index ec49c86268..aa89e466a6 100644
--- a/.github/workflows/remove-graduated-overrides.yml
+++ b/.github/workflows/remove-graduated-overrides.yml
@@ -70,7 +70,7 @@ jobs:
                 -m "Triggered by remove-graduated-overrides GitHub Action."
           fi
       - name: Open pull request
-        uses: peter-evans/create-pull-request@v4.2.3
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
           branch: ${{ matrix.branch }}-graduation
diff --git a/ci/find-whitespace b/ci/find-whitespace
index 05817a5a08..d72b9b7e1c 100755
--- a/ci/find-whitespace
+++ b/ci/find-whitespace
@@ -3,16 +3,62 @@
 set -euo pipefail
 
 main() {
-    local files_with_whitespace
-    files_with_whitespace="$(find manifests overlay.d tests -type f -exec grep -El " +$" {} \;)"
 
+    local files_with_whitespace=""
+    local files_with_missing_empty_line_at_eof=""
+
+    while IFS= read -r -d '' f; do
+        echo "[+] Checking ${f}"
+
+        # Looking for whitespace at end of line
+        if grep -Eq " +$" "${f}"; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./live/isolinux/boot.msg" ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring whitespace at end of line"
+            else
+                echo "[+] Checking ${f}: Found whitespace at end of line"
+                files_with_whitespace+=" ${f}"
+            fi
+        fi
+
+        # Looking for missing empty line at end of file
+        if [[ -n $(tail -c 1 "${f}") ]]; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-gs/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-s3/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/remote/data/expected/"* ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring missing empty line at end of file"
+            else
+                echo "[+] Checking ${f}: Missing empty line at end of file"
+                files_with_missing_empty_line_at_eof+=" ${f}"
+            fi
+        fi
+    done< <(find . -path "./.git" -prune -o -type f -print0)
+
+    echo ""
     if [[ -n "${files_with_whitespace}" ]]; then
         echo "[+] Found files with whitespace at the end of line"
-        echo "${files_with_whitespace}"
+        echo "${files_with_whitespace}" | tr ' ' '\n'
+    else
+        echo "[+] No files with whitespace at the end of line"
+    fi
+
+    echo ""
+    if [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
+        echo "[+] Found files with missing empty line at end of file"
+        echo "${files_with_missing_empty_line_at_eof}" | tr ' ' '\n'
+    else
+        echo "[+] No files with missing empty line at end of file"
+    fi
+
+    if [[ -n "${files_with_whitespace}" ]] || [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
         exit 1
     fi
 
-    echo "[+] No files with whitespace at the end of line"
     exit 0
 }
 
diff --git a/image-base.yaml b/image-base.yaml
index 974433e2d0..cc038b04b8 100644
--- a/image-base.yaml
+++ b/image-base.yaml
@@ -17,15 +17,6 @@ ignition-network-kcmdline: []
 # Optional remote by which to prefix the deployed OSTree ref
 ostree-remote: fedora
 
-# opt in to using the `metadata_csum_seed` feature of the ext4 filesystem
-# for the /boot filesystem. Support for this was only recently added to grub
-# and isn't available everywhere yet so we'll gate it behind this image.yaml
-# knob. It should be easy to know when RHEL/RHCOS supports this by just flipping
-# this to `true` and doing a build. It should error when building the disk
-# images if grub doesn't support it.
-# https://lists.gnu.org/archive/html/grub-devel/2021-06/msg00031.html
-bootfs_metadata_csum_seed: true
-
 vmware-os-type: fedora64Guest
 # VMware hardware versions: https://kb.vmware.com/s/article/1003746
 # We use the newest version allowed by the oldest non-EOL VMware
diff --git a/kola-denylist.yaml b/kola-denylist.yaml
index 7fc118bcd8..789a86db8d 100644
--- a/kola-denylist.yaml
+++ b/kola-denylist.yaml
@@ -7,27 +7,12 @@
   tracker: https://github.com/coreos/coreos-assembler/pull/1478
 - pattern: coreos.ignition.ssh.key
   tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1553
-  # snooze: 2024-04-15 (disabled on promotion)
+  # snooze: 2024-04-29 (disabled on promotion)
   # warn: true (disabled on promotion)
   platforms:
     - azure
-- pattern: ext.config.var-mount.scsi-id
-  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1670
-  # snooze: 2024-04-15 (disabled on promotion)
-  # warn: true (disabled on promotion)
-  streams:
-    - rawhide
-    - branched
-    - next
-    - next-devel
 - pattern: coreos.boot-mirror*
   tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1659
-  warn: true
+  # warn: true (disabled on promotion)
   arches:
     - ppc64le
-- pattern: ext.config.kdump.crash
-  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1698
-  # snooze: 2024-04-07 (disabled on promotion)
-  # warn: true (disabled on promotion)
-  streams:
-    - rawhide
diff --git a/manifest-lock.aarch64.json b/manifest-lock.aarch64.json
index 4e3aeaa80b..755a27efdc 100644
--- a/manifest-lock.aarch64.json
+++ b/manifest-lock.aarch64.json
@@ -73,19 +73,19 @@
       }
     },
     "amd-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "amd-ucode-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "atheros-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -175,7 +175,7 @@
       }
     },
     "brcmfmac-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -211,7 +211,7 @@
       }
     },
     "c-ares": {
-      "evra": "1.25.0-1.fc39.aarch64",
+      "evra": "1.28.1-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "c-ares"
       }
@@ -241,7 +241,7 @@
       }
     },
     "cirrus-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -793,25 +793,25 @@
       }
     },
     "glibc": {
-      "evra": "2.38-17.fc39.aarch64",
+      "evra": "2.38-18.fc39.aarch64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-common": {
-      "evra": "2.38-17.fc39.aarch64",
+      "evra": "2.38-18.fc39.aarch64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-gconv-extra": {
-      "evra": "2.38-17.fc39.aarch64",
+      "evra": "2.38-18.fc39.aarch64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-minimal-langpack": {
-      "evra": "2.38-17.fc39.aarch64",
+      "evra": "2.38-18.fc39.aarch64",
       "metadata": {
         "sourcerpm": "glibc"
       }
@@ -829,7 +829,7 @@
       }
     },
     "gnutls": {
-      "evra": "3.8.4-1.fc39.aarch64",
+      "evra": "3.8.5-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "gnutls"
       }
@@ -907,13 +907,13 @@
       }
     },
     "intel-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "intel-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1015,7 +1015,7 @@
       }
     },
     "jose": {
-      "evra": "11-8.fc39.aarch64",
+      "evra": "13-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1057,25 +1057,25 @@
       }
     },
     "kernel": {
-      "evra": "6.8.4-200.fc39.aarch64",
+      "evra": "6.8.6-200.fc39.aarch64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-core": {
-      "evra": "6.8.4-200.fc39.aarch64",
+      "evra": "6.8.6-200.fc39.aarch64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules": {
-      "evra": "6.8.4-200.fc39.aarch64",
+      "evra": "6.8.6-200.fc39.aarch64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules-core": {
-      "evra": "6.8.4-200.fc39.aarch64",
+      "evra": "6.8.6-200.fc39.aarch64",
       "metadata": {
         "sourcerpm": "kernel"
       }
@@ -1171,7 +1171,7 @@
       }
     },
     "libblkid": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1183,7 +1183,7 @@
       }
     },
     "libbsd": {
-      "evra": "0.11.7-5.fc39.aarch64",
+      "evra": "0.12.2-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "libbsd"
       }
@@ -1261,7 +1261,7 @@
       }
     },
     "libfdisk": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1345,7 +1345,7 @@
       }
     },
     "libjose": {
-      "evra": "11-8.fc39.aarch64",
+      "evra": "13-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1405,7 +1405,7 @@
       }
     },
     "libmount": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1435,7 +1435,7 @@
       }
     },
     "libnfsidmap": {
-      "evra": "1:2.6.4-0.rc5.fc39.aarch64",
+      "evra": "1:2.6.4-0.rc6.fc39.aarch64",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1447,7 +1447,7 @@
       }
     },
     "libnghttp2": {
-      "evra": "1.55.1-4.fc39.aarch64",
+      "evra": "1.55.1-5.fc39.aarch64",
       "metadata": {
         "sourcerpm": "nghttp2"
       }
@@ -1567,13 +1567,13 @@
       }
     },
     "libsmartcols": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "libsmbclient": {
-      "evra": "2:4.19.5-1.fc39.aarch64",
+      "evra": "2:4.19.6-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1687,7 +1687,7 @@
       }
     },
     "libuuid": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1705,7 +1705,7 @@
       }
     },
     "libwbclient": {
-      "evra": "2:4.19.5-1.fc39.aarch64",
+      "evra": "2:4.19.6-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1723,7 +1723,7 @@
       }
     },
     "libxmlb": {
-      "evra": "0.3.17-1.fc39.aarch64",
+      "evra": "0.3.18-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "libxmlb"
       }
@@ -1747,13 +1747,13 @@
       }
     },
     "linux-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "linux-firmware-whence": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1837,7 +1837,7 @@
       }
     },
     "mt7xxx-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1897,7 +1897,7 @@
       }
     },
     "nfs-utils-coreos": {
-      "evra": "1:2.6.4-0.rc5.fc39.aarch64",
+      "evra": "1:2.6.4-0.rc6.fc39.aarch64",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1933,7 +1933,7 @@
       }
     },
     "nvidia-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1945,7 +1945,7 @@
       }
     },
     "nxpwireless-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2035,13 +2035,13 @@
       }
     },
     "passt": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.aarch64",
+      "evra": "0^20240326.g4988e2b-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "passt"
       }
     },
     "passt-selinux": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.noarch",
+      "evra": "0^20240326.g4988e2b-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "passt"
       }
@@ -2155,7 +2155,7 @@
       }
     },
     "qemu-user-static-x86": {
-      "evra": "2:8.1.3-4.fc39.aarch64",
+      "evra": "2:8.1.3-5.fc39.aarch64",
       "metadata": {
         "sourcerpm": "qemu"
       }
@@ -2167,7 +2167,7 @@
       }
     },
     "realtek-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2191,13 +2191,13 @@
       }
     },
     "rpm-ostree": {
-      "evra": "2024.4-3.fc39.aarch64",
+      "evra": "2024.4-6.fc39.aarch64",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
     },
     "rpm-ostree-libs": {
-      "evra": "2024.4-3.fc39.aarch64",
+      "evra": "2024.4-6.fc39.aarch64",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
@@ -2215,7 +2215,7 @@
       }
     },
     "rsync": {
-      "evra": "3.2.7-5.fc39.aarch64",
+      "evra": "3.3.0-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "rsync"
       }
@@ -2227,19 +2227,19 @@
       }
     },
     "samba-client-libs": {
-      "evra": "2:4.19.5-1.fc39.aarch64",
+      "evra": "2:4.19.6-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common": {
-      "evra": "2:4.19.5-1.fc39.noarch",
+      "evra": "2:4.19.6-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common-libs": {
-      "evra": "2:4.19.5-1.fc39.aarch64",
+      "evra": "2:4.19.6-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -2473,7 +2473,7 @@
       }
     },
     "tiwilink-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2515,25 +2515,25 @@
       }
     },
     "util-linux": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "util-linux-core": {
-      "evra": "2.39.3-6.fc39.aarch64",
+      "evra": "2.39.4-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "vim-data": {
-      "evra": "2:9.1.181-1.fc39.noarch",
+      "evra": "2:9.1.309-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "vim"
       }
     },
     "vim-minimal": {
-      "evra": "2:9.1.181-1.fc39.aarch64",
+      "evra": "2:9.1.309-1.fc39.aarch64",
       "metadata": {
         "sourcerpm": "vim"
       }
@@ -2618,16 +2618,16 @@
     }
   },
   "metadata": {
-    "generated": "2024-04-07T00:00:00Z",
+    "generated": "2024-04-21T00:00:00Z",
     "rpmmd_repos": {
       "fedora": {
         "generated": "2023-11-01T00:12:29Z"
       },
       "fedora-coreos-pool": {
-        "generated": "2024-04-07T17:50:51Z"
+        "generated": "2024-04-21T15:19:43Z"
       },
       "fedora-updates": {
-        "generated": "2024-04-07T01:16:43Z"
+        "generated": "2024-04-21T01:11:35Z"
       }
     }
   }
diff --git a/manifest-lock.overrides.yaml b/manifest-lock.overrides.yaml
index 477f35d32c..62cfbe5a1d 100644
--- a/manifest-lock.overrides.yaml
+++ b/manifest-lock.overrides.yaml
@@ -8,16 +8,4 @@
 # in the `metadata.reason` key, though it's acceptable to omit a `reason`
 # for FCOS-specific packages (ignition, afterburn, etc.).
 
-packages:
-  rpm-ostree:
-    evr: 2024.4-6.fc39
-    metadata:
-      bodhi: https://bodhi.fedoraproject.org/updates/FEDORA-2024-4afd3d38ae
-      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1705
-      type: fast-track
-  rpm-ostree-libs:
-    evr: 2024.4-6.fc39
-    metadata:
-      bodhi: https://bodhi.fedoraproject.org/updates/FEDORA-2024-4afd3d38ae
-      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1705
-      type: fast-track
+packages: {}
diff --git a/manifest-lock.ppc64le.json b/manifest-lock.ppc64le.json
index b439e803b8..85261aa18b 100644
--- a/manifest-lock.ppc64le.json
+++ b/manifest-lock.ppc64le.json
@@ -73,19 +73,19 @@
       }
     },
     "amd-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "amd-ucode-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "atheros-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -181,7 +181,7 @@
       }
     },
     "brcmfmac-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -217,7 +217,7 @@
       }
     },
     "c-ares": {
-      "evra": "1.25.0-1.fc39.ppc64le",
+      "evra": "1.28.1-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "c-ares"
       }
@@ -247,7 +247,7 @@
       }
     },
     "cirrus-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -769,25 +769,25 @@
       }
     },
     "glibc": {
-      "evra": "2.38-17.fc39.ppc64le",
+      "evra": "2.38-18.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-common": {
-      "evra": "2.38-17.fc39.ppc64le",
+      "evra": "2.38-18.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-gconv-extra": {
-      "evra": "2.38-17.fc39.ppc64le",
+      "evra": "2.38-18.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-minimal-langpack": {
-      "evra": "2.38-17.fc39.ppc64le",
+      "evra": "2.38-18.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "glibc"
       }
@@ -805,7 +805,7 @@
       }
     },
     "gnutls": {
-      "evra": "3.8.4-1.fc39.ppc64le",
+      "evra": "3.8.5-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "gnutls"
       }
@@ -883,13 +883,13 @@
       }
     },
     "intel-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "intel-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -991,7 +991,7 @@
       }
     },
     "jose": {
-      "evra": "11-8.fc39.ppc64le",
+      "evra": "13-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1033,25 +1033,25 @@
       }
     },
     "kernel": {
-      "evra": "6.8.4-200.fc39.ppc64le",
+      "evra": "6.8.6-200.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-core": {
-      "evra": "6.8.4-200.fc39.ppc64le",
+      "evra": "6.8.6-200.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules": {
-      "evra": "6.8.4-200.fc39.ppc64le",
+      "evra": "6.8.6-200.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules-core": {
-      "evra": "6.8.4-200.fc39.ppc64le",
+      "evra": "6.8.6-200.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "kernel"
       }
@@ -1147,7 +1147,7 @@
       }
     },
     "libblkid": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1159,7 +1159,7 @@
       }
     },
     "libbsd": {
-      "evra": "0.11.7-5.fc39.ppc64le",
+      "evra": "0.12.2-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "libbsd"
       }
@@ -1237,7 +1237,7 @@
       }
     },
     "libfdisk": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1321,7 +1321,7 @@
       }
     },
     "libjose": {
-      "evra": "11-8.fc39.ppc64le",
+      "evra": "13-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1381,7 +1381,7 @@
       }
     },
     "libmount": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1411,7 +1411,7 @@
       }
     },
     "libnfsidmap": {
-      "evra": "1:2.6.4-0.rc5.fc39.ppc64le",
+      "evra": "1:2.6.4-0.rc6.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1423,7 +1423,7 @@
       }
     },
     "libnghttp2": {
-      "evra": "1.55.1-4.fc39.ppc64le",
+      "evra": "1.55.1-5.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "nghttp2"
       }
@@ -1555,13 +1555,13 @@
       }
     },
     "libsmartcols": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "libsmbclient": {
-      "evra": "2:4.19.5-1.fc39.ppc64le",
+      "evra": "2:4.19.6-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1675,7 +1675,7 @@
       }
     },
     "libuuid": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1693,7 +1693,7 @@
       }
     },
     "libwbclient": {
-      "evra": "2:4.19.5-1.fc39.ppc64le",
+      "evra": "2:4.19.6-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1711,7 +1711,7 @@
       }
     },
     "libxmlb": {
-      "evra": "0.3.17-1.fc39.ppc64le",
+      "evra": "0.3.18-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "libxmlb"
       }
@@ -1735,13 +1735,13 @@
       }
     },
     "linux-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "linux-firmware-whence": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1819,7 +1819,7 @@
       }
     },
     "mt7xxx-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1879,7 +1879,7 @@
       }
     },
     "nfs-utils-coreos": {
-      "evra": "1:2.6.4-0.rc5.fc39.ppc64le",
+      "evra": "1:2.6.4-0.rc6.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1915,7 +1915,7 @@
       }
     },
     "nvidia-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1927,7 +1927,7 @@
       }
     },
     "nxpwireless-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2023,13 +2023,13 @@
       }
     },
     "passt": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.ppc64le",
+      "evra": "0^20240326.g4988e2b-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "passt"
       }
     },
     "passt-selinux": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.noarch",
+      "evra": "0^20240326.g4988e2b-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "passt"
       }
@@ -2155,7 +2155,7 @@
       }
     },
     "qemu-user-static-x86": {
-      "evra": "2:8.1.3-4.fc39.ppc64le",
+      "evra": "2:8.1.3-5.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "qemu"
       }
@@ -2167,7 +2167,7 @@
       }
     },
     "realtek-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2191,13 +2191,13 @@
       }
     },
     "rpm-ostree": {
-      "evra": "2024.4-3.fc39.ppc64le",
+      "evra": "2024.4-6.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
     },
     "rpm-ostree-libs": {
-      "evra": "2024.4-3.fc39.ppc64le",
+      "evra": "2024.4-6.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
@@ -2215,7 +2215,7 @@
       }
     },
     "rsync": {
-      "evra": "3.2.7-5.fc39.ppc64le",
+      "evra": "3.3.0-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "rsync"
       }
@@ -2227,19 +2227,19 @@
       }
     },
     "samba-client-libs": {
-      "evra": "2:4.19.5-1.fc39.ppc64le",
+      "evra": "2:4.19.6-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common": {
-      "evra": "2:4.19.5-1.fc39.noarch",
+      "evra": "2:4.19.6-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common-libs": {
-      "evra": "2:4.19.5-1.fc39.ppc64le",
+      "evra": "2:4.19.6-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -2467,7 +2467,7 @@
       }
     },
     "tiwilink-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2509,25 +2509,25 @@
       }
     },
     "util-linux": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "util-linux-core": {
-      "evra": "2.39.3-6.fc39.ppc64le",
+      "evra": "2.39.4-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "vim-data": {
-      "evra": "2:9.1.181-1.fc39.noarch",
+      "evra": "2:9.1.309-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "vim"
       }
     },
     "vim-minimal": {
-      "evra": "2:9.1.181-1.fc39.ppc64le",
+      "evra": "2:9.1.309-1.fc39.ppc64le",
       "metadata": {
         "sourcerpm": "vim"
       }
@@ -2606,16 +2606,16 @@
     }
   },
   "metadata": {
-    "generated": "2024-04-07T00:00:00Z",
+    "generated": "2024-04-21T00:00:00Z",
     "rpmmd_repos": {
       "fedora": {
         "generated": "2023-11-01T00:12:28Z"
       },
       "fedora-coreos-pool": {
-        "generated": "2024-04-07T17:49:39Z"
+        "generated": "2024-04-21T15:17:53Z"
       },
       "fedora-updates": {
-        "generated": "2024-04-07T01:16:51Z"
+        "generated": "2024-04-21T01:11:44Z"
       }
     }
   }
diff --git a/manifest-lock.s390x.json b/manifest-lock.s390x.json
index a32be14dbe..dfcd5e45e6 100644
--- a/manifest-lock.s390x.json
+++ b/manifest-lock.s390x.json
@@ -73,19 +73,19 @@
       }
     },
     "amd-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "amd-ucode-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "atheros-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -175,7 +175,7 @@
       }
     },
     "brcmfmac-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -211,7 +211,7 @@
       }
     },
     "c-ares": {
-      "evra": "1.25.0-1.fc39.s390x",
+      "evra": "1.28.1-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "c-ares"
       }
@@ -241,7 +241,7 @@
       }
     },
     "cirrus-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -745,25 +745,25 @@
       }
     },
     "glibc": {
-      "evra": "2.38-17.fc39.s390x",
+      "evra": "2.38-18.fc39.s390x",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-common": {
-      "evra": "2.38-17.fc39.s390x",
+      "evra": "2.38-18.fc39.s390x",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-gconv-extra": {
-      "evra": "2.38-17.fc39.s390x",
+      "evra": "2.38-18.fc39.s390x",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-minimal-langpack": {
-      "evra": "2.38-17.fc39.s390x",
+      "evra": "2.38-18.fc39.s390x",
       "metadata": {
         "sourcerpm": "glibc"
       }
@@ -781,7 +781,7 @@
       }
     },
     "gnutls": {
-      "evra": "3.8.4-1.fc39.s390x",
+      "evra": "3.8.5-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "gnutls"
       }
@@ -829,13 +829,13 @@
       }
     },
     "intel-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "intel-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -931,7 +931,7 @@
       }
     },
     "jose": {
-      "evra": "11-8.fc39.s390x",
+      "evra": "13-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -973,25 +973,25 @@
       }
     },
     "kernel": {
-      "evra": "6.8.4-200.fc39.s390x",
+      "evra": "6.8.6-200.fc39.s390x",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-core": {
-      "evra": "6.8.4-200.fc39.s390x",
+      "evra": "6.8.6-200.fc39.s390x",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules": {
-      "evra": "6.8.4-200.fc39.s390x",
+      "evra": "6.8.6-200.fc39.s390x",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules-core": {
-      "evra": "6.8.4-200.fc39.s390x",
+      "evra": "6.8.6-200.fc39.s390x",
       "metadata": {
         "sourcerpm": "kernel"
       }
@@ -1087,7 +1087,7 @@
       }
     },
     "libblkid": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1099,7 +1099,7 @@
       }
     },
     "libbsd": {
-      "evra": "0.11.7-5.fc39.s390x",
+      "evra": "0.12.2-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "libbsd"
       }
@@ -1177,7 +1177,7 @@
       }
     },
     "libfdisk": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1261,7 +1261,7 @@
       }
     },
     "libjose": {
-      "evra": "11-8.fc39.s390x",
+      "evra": "13-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1321,7 +1321,7 @@
       }
     },
     "libmount": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1351,7 +1351,7 @@
       }
     },
     "libnfsidmap": {
-      "evra": "1:2.6.4-0.rc5.fc39.s390x",
+      "evra": "1:2.6.4-0.rc6.fc39.s390x",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1363,7 +1363,7 @@
       }
     },
     "libnghttp2": {
-      "evra": "1.55.1-4.fc39.s390x",
+      "evra": "1.55.1-5.fc39.s390x",
       "metadata": {
         "sourcerpm": "nghttp2"
       }
@@ -1483,13 +1483,13 @@
       }
     },
     "libsmartcols": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "libsmbclient": {
-      "evra": "2:4.19.5-1.fc39.s390x",
+      "evra": "2:4.19.6-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1603,7 +1603,7 @@
       }
     },
     "libuuid": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1621,7 +1621,7 @@
       }
     },
     "libwbclient": {
-      "evra": "2:4.19.5-1.fc39.s390x",
+      "evra": "2:4.19.6-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1639,7 +1639,7 @@
       }
     },
     "libxmlb": {
-      "evra": "0.3.17-1.fc39.s390x",
+      "evra": "0.3.18-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "libxmlb"
       }
@@ -1663,13 +1663,13 @@
       }
     },
     "linux-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "linux-firmware-whence": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1747,7 +1747,7 @@
       }
     },
     "mt7xxx-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1807,7 +1807,7 @@
       }
     },
     "nfs-utils-coreos": {
-      "evra": "1:2.6.4-0.rc5.fc39.s390x",
+      "evra": "1:2.6.4-0.rc6.fc39.s390x",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1837,7 +1837,7 @@
       }
     },
     "nvidia-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1849,7 +1849,7 @@
       }
     },
     "nxpwireless-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1933,13 +1933,13 @@
       }
     },
     "passt": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.s390x",
+      "evra": "0^20240326.g4988e2b-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "passt"
       }
     },
     "passt-selinux": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.noarch",
+      "evra": "0^20240326.g4988e2b-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "passt"
       }
@@ -2053,7 +2053,7 @@
       }
     },
     "qemu-user-static-x86": {
-      "evra": "2:8.1.3-4.fc39.s390x",
+      "evra": "2:8.1.3-5.fc39.s390x",
       "metadata": {
         "sourcerpm": "qemu"
       }
@@ -2065,7 +2065,7 @@
       }
     },
     "realtek-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2089,13 +2089,13 @@
       }
     },
     "rpm-ostree": {
-      "evra": "2024.4-3.fc39.s390x",
+      "evra": "2024.4-6.fc39.s390x",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
     },
     "rpm-ostree-libs": {
-      "evra": "2024.4-3.fc39.s390x",
+      "evra": "2024.4-6.fc39.s390x",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
@@ -2113,7 +2113,7 @@
       }
     },
     "rsync": {
-      "evra": "3.2.7-5.fc39.s390x",
+      "evra": "3.3.0-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "rsync"
       }
@@ -2131,19 +2131,19 @@
       }
     },
     "samba-client-libs": {
-      "evra": "2:4.19.5-1.fc39.s390x",
+      "evra": "2:4.19.6-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common": {
-      "evra": "2:4.19.5-1.fc39.noarch",
+      "evra": "2:4.19.6-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common-libs": {
-      "evra": "2:4.19.5-1.fc39.s390x",
+      "evra": "2:4.19.6-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -2365,7 +2365,7 @@
       }
     },
     "tiwilink-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2407,13 +2407,13 @@
       }
     },
     "util-linux": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "util-linux-core": {
-      "evra": "2.39.3-6.fc39.s390x",
+      "evra": "2.39.4-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -2425,13 +2425,13 @@
       }
     },
     "vim-data": {
-      "evra": "2:9.1.181-1.fc39.noarch",
+      "evra": "2:9.1.309-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "vim"
       }
     },
     "vim-minimal": {
-      "evra": "2:9.1.181-1.fc39.s390x",
+      "evra": "2:9.1.309-1.fc39.s390x",
       "metadata": {
         "sourcerpm": "vim"
       }
@@ -2510,16 +2510,16 @@
     }
   },
   "metadata": {
-    "generated": "2024-04-07T00:00:00Z",
+    "generated": "2024-04-21T00:00:00Z",
     "rpmmd_repos": {
       "fedora": {
         "generated": "2023-11-01T00:12:19Z"
       },
       "fedora-coreos-pool": {
-        "generated": "2024-04-07T17:49:20Z"
+        "generated": "2024-04-21T15:17:00Z"
       },
       "fedora-updates": {
-        "generated": "2024-04-07T01:16:59Z"
+        "generated": "2024-04-21T01:11:53Z"
       }
     }
   }
diff --git a/manifest-lock.x86_64.json b/manifest-lock.x86_64.json
index c3e4a1f3aa..7ccc9500b9 100644
--- a/manifest-lock.x86_64.json
+++ b/manifest-lock.x86_64.json
@@ -73,19 +73,19 @@
       }
     },
     "amd-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "amd-ucode-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "atheros-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -175,7 +175,7 @@
       }
     },
     "brcmfmac-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -211,7 +211,7 @@
       }
     },
     "c-ares": {
-      "evra": "1.25.0-1.fc39.x86_64",
+      "evra": "1.28.1-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "c-ares"
       }
@@ -241,7 +241,7 @@
       }
     },
     "cirrus-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -793,25 +793,25 @@
       }
     },
     "glibc": {
-      "evra": "2.38-17.fc39.x86_64",
+      "evra": "2.38-18.fc39.x86_64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-common": {
-      "evra": "2.38-17.fc39.x86_64",
+      "evra": "2.38-18.fc39.x86_64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-gconv-extra": {
-      "evra": "2.38-17.fc39.x86_64",
+      "evra": "2.38-18.fc39.x86_64",
       "metadata": {
         "sourcerpm": "glibc"
       }
     },
     "glibc-minimal-langpack": {
-      "evra": "2.38-17.fc39.x86_64",
+      "evra": "2.38-18.fc39.x86_64",
       "metadata": {
         "sourcerpm": "glibc"
       }
@@ -829,7 +829,7 @@
       }
     },
     "gnutls": {
-      "evra": "3.8.4-1.fc39.x86_64",
+      "evra": "3.8.5-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "gnutls"
       }
@@ -919,13 +919,13 @@
       }
     },
     "intel-audio-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "intel-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1027,7 +1027,7 @@
       }
     },
     "jose": {
-      "evra": "11-8.fc39.x86_64",
+      "evra": "13-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1069,25 +1069,25 @@
       }
     },
     "kernel": {
-      "evra": "6.8.4-200.fc39.x86_64",
+      "evra": "6.8.6-200.fc39.x86_64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-core": {
-      "evra": "6.8.4-200.fc39.x86_64",
+      "evra": "6.8.6-200.fc39.x86_64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules": {
-      "evra": "6.8.4-200.fc39.x86_64",
+      "evra": "6.8.6-200.fc39.x86_64",
       "metadata": {
         "sourcerpm": "kernel"
       }
     },
     "kernel-modules-core": {
-      "evra": "6.8.4-200.fc39.x86_64",
+      "evra": "6.8.6-200.fc39.x86_64",
       "metadata": {
         "sourcerpm": "kernel"
       }
@@ -1183,7 +1183,7 @@
       }
     },
     "libblkid": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1195,7 +1195,7 @@
       }
     },
     "libbsd": {
-      "evra": "0.11.7-5.fc39.x86_64",
+      "evra": "0.12.2-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "libbsd"
       }
@@ -1273,7 +1273,7 @@
       }
     },
     "libfdisk": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1357,7 +1357,7 @@
       }
     },
     "libjose": {
-      "evra": "11-8.fc39.x86_64",
+      "evra": "13-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "jose"
       }
@@ -1417,7 +1417,7 @@
       }
     },
     "libmount": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1447,7 +1447,7 @@
       }
     },
     "libnfsidmap": {
-      "evra": "1:2.6.4-0.rc5.fc39.x86_64",
+      "evra": "1:2.6.4-0.rc6.fc39.x86_64",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1459,7 +1459,7 @@
       }
     },
     "libnghttp2": {
-      "evra": "1.55.1-4.fc39.x86_64",
+      "evra": "1.55.1-5.fc39.x86_64",
       "metadata": {
         "sourcerpm": "nghttp2"
       }
@@ -1579,13 +1579,13 @@
       }
     },
     "libsmartcols": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "libsmbclient": {
-      "evra": "2:4.19.5-1.fc39.x86_64",
+      "evra": "2:4.19.6-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1699,7 +1699,7 @@
       }
     },
     "libuuid": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
@@ -1717,7 +1717,7 @@
       }
     },
     "libwbclient": {
-      "evra": "2:4.19.5-1.fc39.x86_64",
+      "evra": "2:4.19.6-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -1735,7 +1735,7 @@
       }
     },
     "libxmlb": {
-      "evra": "0.3.17-1.fc39.x86_64",
+      "evra": "0.3.18-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "libxmlb"
       }
@@ -1759,13 +1759,13 @@
       }
     },
     "linux-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
     },
     "linux-firmware-whence": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1855,7 +1855,7 @@
       }
     },
     "mt7xxx-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1915,7 +1915,7 @@
       }
     },
     "nfs-utils-coreos": {
-      "evra": "1:2.6.4-0.rc5.fc39.x86_64",
+      "evra": "1:2.6.4-0.rc6.fc39.x86_64",
       "metadata": {
         "sourcerpm": "nfs-utils"
       }
@@ -1951,7 +1951,7 @@
       }
     },
     "nvidia-gpu-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -1963,7 +1963,7 @@
       }
     },
     "nxpwireless-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2053,13 +2053,13 @@
       }
     },
     "passt": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.x86_64",
+      "evra": "0^20240326.g4988e2b-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "passt"
       }
     },
     "passt-selinux": {
-      "evra": "0^20240220.g1e6f92b-1.fc39.noarch",
+      "evra": "0^20240326.g4988e2b-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "passt"
       }
@@ -2179,7 +2179,7 @@
       }
     },
     "realtek-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2203,13 +2203,13 @@
       }
     },
     "rpm-ostree": {
-      "evra": "2024.4-3.fc39.x86_64",
+      "evra": "2024.4-6.fc39.x86_64",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
     },
     "rpm-ostree-libs": {
-      "evra": "2024.4-3.fc39.x86_64",
+      "evra": "2024.4-6.fc39.x86_64",
       "metadata": {
         "sourcerpm": "rpm-ostree"
       }
@@ -2227,7 +2227,7 @@
       }
     },
     "rsync": {
-      "evra": "3.2.7-5.fc39.x86_64",
+      "evra": "3.3.0-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "rsync"
       }
@@ -2239,19 +2239,19 @@
       }
     },
     "samba-client-libs": {
-      "evra": "2:4.19.5-1.fc39.x86_64",
+      "evra": "2:4.19.6-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common": {
-      "evra": "2:4.19.5-1.fc39.noarch",
+      "evra": "2:4.19.6-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "samba"
       }
     },
     "samba-common-libs": {
-      "evra": "2:4.19.5-1.fc39.x86_64",
+      "evra": "2:4.19.6-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "samba"
       }
@@ -2485,7 +2485,7 @@
       }
     },
     "tiwilink-firmware": {
-      "evra": "20240312-1.fc39.noarch",
+      "evra": "20240410-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "linux-firmware"
       }
@@ -2527,25 +2527,25 @@
       }
     },
     "util-linux": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "util-linux-core": {
-      "evra": "2.39.3-6.fc39.x86_64",
+      "evra": "2.39.4-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "util-linux"
       }
     },
     "vim-data": {
-      "evra": "2:9.1.181-1.fc39.noarch",
+      "evra": "2:9.1.309-1.fc39.noarch",
       "metadata": {
         "sourcerpm": "vim"
       }
     },
     "vim-minimal": {
-      "evra": "2:9.1.181-1.fc39.x86_64",
+      "evra": "2:9.1.309-1.fc39.x86_64",
       "metadata": {
         "sourcerpm": "vim"
       }
@@ -2630,16 +2630,16 @@
     }
   },
   "metadata": {
-    "generated": "2024-04-07T00:00:00Z",
+    "generated": "2024-04-21T00:00:00Z",
     "rpmmd_repos": {
       "fedora": {
         "generated": "2023-11-01T00:12:39Z"
       },
       "fedora-coreos-pool": {
-        "generated": "2024-04-07T17:50:50Z"
+        "generated": "2024-04-21T15:19:06Z"
       },
       "fedora-updates": {
-        "generated": "2024-04-07T01:17:08Z"
+        "generated": "2024-04-21T01:12:02Z"
       }
     }
   }
diff --git a/manifests/fedora-coreos.yaml b/manifests/fedora-coreos.yaml
index ada7419b8a..de30f75895 100644
--- a/manifests/fedora-coreos.yaml
+++ b/manifests/fedora-coreos.yaml
@@ -34,8 +34,8 @@ conditional-include:
     # Checks for breaking changes that came with Podman v5.
     include: podman-v5.yaml
   - if: releasever >= 41
-    # Include makedumpfile subpackage from kexec-tools (new in F41+)
-    include: makedumpfile.yaml
+    # Include makedumpfile and kdump-utils subpackages from kexec-tools (new in F41+)
+    include: kdump.yaml
   # On <41, we want to keep making sure dnf doesn't slip in somehow
   # On 41+, we do want it
   # https://github.com/coreos/fedora-coreos-tracker/issues/1687
diff --git a/manifests/kdump.yaml b/manifests/kdump.yaml
new file mode 100644
index 0000000000..cee3521c68
--- /dev/null
+++ b/manifests/kdump.yaml
@@ -0,0 +1,8 @@
+packages:
+# makedumpfile is needed for kdump and was broken out to a subpackage
+# in F41+. Move this to a main manifest file when we no longer support <F41.
+  - makedumpfile
+# systemd service files and other tools were moved into a subpackage
+# in F41+. Move this to a main manifest file when we no longer support <F41.
+# https://src.fedoraproject.org/rpms/kexec-tools/c/372b4c6c1597c3d9aaddda2d691260474973989e?branch=rawhide
+  - kdump-utils
diff --git a/manifests/makedumpfile.yaml b/manifests/makedumpfile.yaml
deleted file mode 100644
index 5d5813df50..0000000000
--- a/manifests/makedumpfile.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# makedumpfile is needed for kdump and was broken out to a subpackage
-# in F41+. Drop this when we no longer support <F41.
-packages:
-  - makedumpfile
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
index a9bff32d30..2ba3dee7f9 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
@@ -27,8 +27,7 @@ EARLY_DIR="${2:-/tmp}"
 
 IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
 cmdline_arg() {
-    local name value
-    name="$1"
+    local name="$1" value=""
     for arg in "${cmdline[@]}"; do
         if [[ "${arg%%=*}" == "${name}" ]]; then
             value="${arg#*=}"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
index 793e519232..9c4eee089a 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
@@ -1 +1 @@
-`00-core.ign` is the base config shared between FCOS and RHCOS. The configs specific to FCOS are in [50ignition-conf-fcos](../../../../../../15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos).
\ No newline at end of file
+`00-core.ign` is the base config shared between FCOS and RHCOS. The configs specific to FCOS are in [50ignition-conf-fcos](../../../../../../15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos).
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
index ce25509ac6..3674431ff8 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
@@ -12,4 +12,4 @@ Type=oneshot
 ExecStart=/usr/sbin/ignition-ostree-growfs
 RemainAfterExit=yes
 # So we can transiently mount sysroot
-MountFlags=slave
\ No newline at end of file
+MountFlags=slave
diff --git a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator
index a38da85a01..9edb7a88eb 100755
--- a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator
+++ b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator
@@ -63,4 +63,4 @@ if have_some_karg 'systemd.unit=rescue.target' rescue single s S 1; then
     write_dropin rescue
 elif have_some_karg 'systemd.unit=emergency.target' emergency '-b' ; then
     write_dropin emergency
-fi
\ No newline at end of file
+fi
diff --git a/tests/kola/containers/quadlet/test.sh b/tests/kola/containers/quadlet/test.sh
index aa6bcc78ff..ae597123a5 100755
--- a/tests/kola/containers/quadlet/test.sh
+++ b/tests/kola/containers/quadlet/test.sh
@@ -9,23 +9,36 @@ set -xeuo pipefail
 # shellcheck disable=SC1091
 . "$KOLA_EXT_DATA/commonlib.sh"
 
-if [[ "$(podman volume inspect systemd-test | jq -r '.[0].Labels."org.test.Key"')" != "quadlet-test-volume" ]]; then
+# Test volume
+if ! is_service_active test-volume.service; then
+  fatal "test-volume.service failed to start"
+fi
+volume_info=$(podman volume inspect systemd-test)
+if [[ "$(jq -r '.[0].Labels."org.test.Key"' <<< "$volume_info")" != "quadlet-test-volume" ]]; then
     fatal "Volume not correctly created"
 fi
 
-if [[ "$(podman network inspect systemd-test | jq -r '.[0].labels."org.test.Key"')" != "quadlet-test-network" ]]; then
+# Test network
+if ! is_service_active test-network.service; then
+  fatal "test-network.service failed to start"
+fi
+network_info=$(podman network inspect systemd-test)
+if [[ "$(jq -r '.[0].labels."org.test.Key"' <<< "$network_info")" != "quadlet-test-network" ]]; then
     fatal "Network not correctly created"
 fi
 
-if [[ "$(podman inspect systemd-test | jq -r '.[0].ImageName')" != "quay.io/fedora/fedora-minimal:39" ]]; then
+# Test container
+if ! is_service_active test.service; then
+  fatal "test-network.service failed to start"
+fi
+container_info=$(podman container inspect systemd-test)
+if [[ "$(jq -r '.[0].ImageName' <<< "$container_info")" != "quay.io/fedora/fedora-minimal:39" ]]; then
     fatal "Container not using the correct image"
 fi
-
-if [[ "$(podman inspect systemd-test | jq -r '.[0].NetworkSettings.Networks[].NetworkID')" != "systemd-test" ]]; then
+if [[ "$(jq -r '.[0].NetworkSettings.Networks[].NetworkID' <<< "$container_info")" != "systemd-test" ]]; then
     fatal "Container not using the correct network"
 fi
-
-if [[ "$(podman inspect systemd-test | jq -r '.[0].HostConfig.Binds[0]')" != "systemd-test:/data:rw,rprivate,nosuid,nodev,rbind" ]]; then
+if [[ "$(jq -r '.[0].HostConfig.Binds[0]' <<< "$container_info")" != "systemd-test:/data:rw,rprivate,nosuid,nodev,rbind" ]]; then
     fatal "Container not using the correct volume"
 fi
 
diff --git a/tests/kola/kubernetes/systemd-env-read/config.bu b/tests/kola/kubernetes/systemd-env-read/config.bu
index 68927604e8..84f5bd89e7 100644
--- a/tests/kola/kubernetes/systemd-env-read/config.bu
+++ b/tests/kola/kubernetes/systemd-env-read/config.bu
@@ -24,4 +24,4 @@ systemd:
         RemainAfterExit=yes
         Type=oneshot
         [Install]
-        WantedBy=multi-user.target
\ No newline at end of file
+        WantedBy=multi-user.target
diff --git a/tests/kola/var-mount/scsi-id/config.bu b/tests/kola/var-mount/scsi-id/config.bu
index 97677034a1..118f4215f8 100644
--- a/tests/kola/var-mount/scsi-id/config.bu
+++ b/tests/kola/var-mount/scsi-id/config.bu
@@ -2,7 +2,8 @@ variant: fcos
 version: 1.4.0
 storage:
   disks:
-    - device: /dev/disk/by-id/scsi-0NVME_VirtualMultipath_disk1
+    # the symlink is from wwn=123456789 converted to hexadecimal (base16)
+    - device: /dev/disk/by-id/scsi-300000000075bcd15
       wipe_table: true
       partitions:
         - number: 1
@@ -13,4 +14,4 @@ storage:
       format: xfs
       wipe_filesystem: true
       label: var
-      with_mount_unit: true
\ No newline at end of file
+      with_mount_unit: true
diff --git a/tests/kola/var-mount/scsi-id/test.sh b/tests/kola/var-mount/scsi-id/test.sh
index 4885786856..405f7a0f55 100755
--- a/tests/kola/var-mount/scsi-id/test.sh
+++ b/tests/kola/var-mount/scsi-id/test.sh
@@ -2,7 +2,7 @@
 ## kola:
 ##   # additionalDisks is only supported on QEMU
 ##   platforms: qemu
-##   additionalDisks: ["5G:mpath"]
+##   additionalDisks: ["5G:channel=scsi,wwn=123456789"]
 ##   description: Verify udev rules /dev/disk/by-id/scsi-* symlinks exist
 ##     in initramfs.
 
diff --git a/tests/manual/coreos-builds-bisect.py b/tests/manual/coreos-builds-bisect.py
new file mode 100755
index 0000000000..49a27a9f00
--- /dev/null
+++ b/tests/manual/coreos-builds-bisect.py
@@ -0,0 +1,303 @@
+#!/usr/bin/python3
+#
+# Copyright 2024 Dusty Mabe <dusty@dustymabe.com>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This program will bisect a list of CoreOS builds, like what are
+# stored in a builds.json file. Example:
+#
+# It was heavily inspired by rpm-ostree-bisect:
+# https://github.com/ostreedev/ostree-releng-scripts/blob/master/rpm-ostree-bisect
+#
+# Here is some high level representation of what it does:
+#
+#    grab info on every build in builds.json
+#    A -> B -> C -> D -> E -> F -> G
+#
+#    user provided good/bad builds
+#
+#    run test script
+#    returns 0 for pass and 1 for failure
+#
+#    known good is A, known bad is G
+#
+#    start bisect:
+#    deploy D, test --> bad
+#    mark D, E, F bad
+#
+#    deploy B, test --> good
+#    mark B good
+#
+#    deploy C, test --> bad
+#
+#    Failure introduced in B -> C
+#
+# An example invocation from a cosa build dir looks like:
+#
+#   cosa buildfetch --force --stream=rawhide # to populate builds/builds.json
+#   ./coreos-builds-bisect.py --bad 41.20240404.91.0  \
+#                             --good 39.20230813.91.0 \
+#                             --testscript=./test.sh
+#
+# If the testing gets interrupted for some reason you should
+# be able to continue it with:
+#
+#   ./coreos-builds-bisect.py --resume                \
+#                             --bad 41.20240404.91.0  \
+#                             --good 39.20230813.91.0 \
+#                             --testscript=./test.sh
+#
+# The testcript that gets called from this will get passed one
+# argument, which is the ID of the build that should be tested.
+# It is up to the test script to pull anything necessary to test
+# that build, run the test, and return success or failure.
+#
+# For example:
+# ```
+#     cat ./test.sh
+#     #!/bin/bash
+#     set -eux -o pipefail
+#     build=$1
+#     cosa buildfetch --force --stream=rawhide --build=$build --artifact=qemu
+#     cosa decompress --build=$build
+#     cosa kola run --build=$build ext.config.mynewtestiwrote
+
+import argparse
+import json
+import os
+import os.path
+import subprocess
+import sys
+import tempfile
+from collections import OrderedDict
+
+DATA_FILE = './coreos-builds-bisect-data.json'
+BUILDS_JSON_FILE = './builds/builds.json'
+
+# Inspired by https://stackoverflow.com/a/287944
+class colors:
+    CLEAR = '\033[0m'
+    CYAN = '\033[96m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RED = '\033[91m'
+
+def fatal(msg):
+    print(colors.RED + msg + colors.CLEAR, file=sys.stderr)
+    sys.exit(1)
+
+def log_color(msg, color):
+    print(color + msg + colors.CLEAR)
+    sys.stdout.flush()
+
+def log(msg):
+    log_color(msg, colors.CYAN)
+
+def log_success(msg):
+    log_color(msg, colors.GREEN)
+
+def log_warn(msg):
+    log_color(msg, colors.YELLOW)
+
+"""
+    Initialize build ID order dict. The array will be a list of
+    commits in descending order. Each entry will be a dict with
+    key of commitid and value = a dict of version, heuristic
+    (TESTED, GIVEN, ASSUMED), and status (GOOD/BAD/UNKNOWN)
+
+    builds = {
+        '39.20230813.91.0' => {
+                      'heuristic', 'TESTED',
+                      'status': 'GOOD',
+                    },
+    }
+"""
+def initialize_builds_info(buildsjson, arch, badbuild, goodbuild):
+    with open(buildsjson, 'r') as f:
+        builds = json.load(f, object_pairs_hook=OrderedDict)
+    # Further narrow in on just the list of build dicts
+    builds = builds['builds']
+
+    # An ordered dictionary of builds info
+    info = OrderedDict()
+
+    # Populate the info dict
+    for build in builds:
+        # If this build has an entry for this architecture add it to
+        # our builds info dict.
+        if arch in build['arches']:
+            info.update({ build['id']: { 'status': 'UNKNOWN',
+                                         'heuristic': 'ASSUMED'}})
+    # Mark the bad commit bad and the good commit good
+    info[badbuild]['status'] = 'BAD'
+    info[badbuild]['heuristic'] = 'GIVEN'
+    info[goodbuild]['status'] = 'GOOD'
+    info[goodbuild]['heuristic'] = 'GIVEN'
+    return info
+
+
+def load_data(datafile):
+    with open(datafile, 'r') as f:
+        data = json.load(f, object_pairs_hook=OrderedDict)
+    return data
+
+
+def write_data(datafile, data):
+    dirname = os.path.dirname(datafile)
+    (_, tmpfile) = tempfile.mkstemp(
+                    dir=dirname,
+                    prefix="coreos-builds-bisect")
+
+    with open(tmpfile, 'w') as f:
+        json.dump(data, f, indent=4)
+    os.rename(tmpfile, datafile)
+
+
+def verify_script(testscript):
+    # Verify test script exists and is executable
+    if not testscript:
+        fatal("Must provide a --testscript to run")
+    if not (os.path.isfile(testscript)
+         and os.access(testscript, os.X_OK)):
+        fatal(f"provided test script: {testscript} is not an executable file")
+
+
+def bisect(args):
+    badbuild  = args.badbuild
+    goodbuild = args.goodbuild
+    testscript = args.testscript
+    datafile = args.datafile
+
+    builds_info = load_data(datafile)
+
+    # Loop until we're done bisecting
+    while True:
+        # Find list of unknown status builds
+        unknowns = []
+        lastbad = None
+        firstgood = None
+        for buildid in builds_info.keys():
+            status = builds_info[buildid]['status']
+            if status == 'BAD':
+                lastbad = buildid
+            elif status == 'UNKNOWN':
+                unknowns.append(buildid)
+            elif status == 'GOOD':
+                firstgood = buildid
+                break
+
+        # If we have no unknowns then we're done!
+        if len(unknowns) == 0:
+            log("BISECT TEST RESULTS:")
+            if firstgood is None:
+                log("No good builds were found in the history!")
+                return 0
+            # Do a sanity check to see if the good commit was actually tested.
+            if builds_info[firstgood]['heuristic'] == 'GIVEN':
+                log_warn("WARNING: The good build detected was the one given by the user.")
+                log_warn("WARNING: Are you sure this build is good?")
+            log(f"Last known good build: {firstgood}")
+            log(f"First known bad build: {lastbad}")
+            return 0
+
+        # Bisect to find new build id to test and then run the test
+        # //2 makes sure to give us an integer rather than a float
+        newbuildid = unknowns[len(unknowns)//2]
+        revisions = len(unknowns)
+        steps = 0
+        while revisions > 0:
+            revisions //= 2
+            steps += 1
+        log(f"Executing test for new build: {newbuildid}. ~{steps} steps left")
+        ec = subprocess.call([testscript, newbuildid])
+        if ec == 0:
+            log_success(f"{newbuildid} passed")
+            success = True
+        else:
+            log_warn(f"{newbuildid} failed")
+            success = False
+
+
+        # update the data with the results
+        if success:
+            for b in reversed(builds_info.keys()):
+                builds_info[b]['status'] = 'GOOD'
+                if b == newbuildid:
+                    break
+        else:
+            for b in builds_info.keys():
+                builds_info[b]['status'] = 'BAD'
+                if b == newbuildid:
+                    break
+        builds_info[b]['heuristic'] = 'TESTED'
+
+        # Save the state in case the script gets interrupted
+        write_data(datafile, builds_info)
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--arch", dest='arch',
+                        help="What architecture to target", action='store')
+    parser.add_argument("--buildsjson", dest='buildsjson',
+                        help="Path to builds.json file",
+                        action='store', default=BUILDS_JSON_FILE)
+    parser.add_argument("--bad", dest='badbuild',
+                        help="Known Bad Build", action='store')
+    parser.add_argument("--good", dest='goodbuild',
+                        help="Known Good Build", action='store')
+    parser.add_argument("--testscript", help="A test script to run",
+                        action='store')
+    parser.add_argument("--resume", help="Resume a running bisection",
+                        action='store_true')
+    parser.add_argument("--datafile", help="data file to use for state",
+                        action='store', default=DATA_FILE)
+    args = parser.parse_args()
+    log(f"Using data file at: {args.datafile}")
+
+    verify_script(args.testscript)
+
+    if not args.arch:
+        cp = subprocess.run(['arch'], capture_output=True)
+        args.arch = cp.stdout.decode('utf-8').strip()
+    log(f"Targetting architecture: {args.arch}")
+
+    if args.resume:
+        if not os.path.exists(args.datafile):
+            fatal(f"Datafile at {args.datafile} must pre-exist with --resume")
+    else:
+        if not args.badbuild or not args.goodbuild:
+            fatal("Must specify both a bad and a good build via --bad and --good")
+        if not os.path.exists(args.buildsjson):
+            fatal(f"A builds.json file does not exist at {args.buildsjson}")
+        if os.path.exists(args.datafile) and not args.resume:
+            log(f"A datafile exists at {args.datafile} but --resume not specified")
+            log("If you want to resume a bisect pass --resume")
+            fatal("If you want to start a new bisect delete the datafile")
+
+        # initialize data
+        builds_info = initialize_builds_info(args.buildsjson,
+                                             args.arch,
+                                             args.badbuild,
+                                             args.goodbuild)
+        # Write data to file
+        write_data(args.datafile, builds_info)
+
+    bisect(args)
+
+if __name__ == '__main__':
+    main()