WIP: Support for install-to-filesystem --replace=alongside

I was trying to be really ambitious in containers#78
for the full "takeover" path.  This is a *much* *much* simpler variant
where we just:

- Blow away and reinitialize the `/boot` and `/boot/efi` partitions
- Write inside the existing filesystem, leaving the OS running

Then when we reboot, we'll just need to clean up the old OS
state (or optionally leave it).

lib/src/bootloader.rs

@@ -75,7 +75,7 @@ pub(crate) fn install_via_bootupd(
     let bootfs = &rootfs.join("boot");
     let bootfs = Dir::open_ambient_dir(bootfs, cap_std::ambient_authority())?;
 
-    {
+    if super::install::ARCH_USES_EFI {
         let efidir = bootfs.open_dir("efi")?;
         install_grub2_efi(&efidir, &grub2_uuid_contents)?;
     }

lib/src/install.rs

@@ -21,6 +21,7 @@ use camino::Utf8PathBuf;
 use cap_std::fs::Dir;
 use cap_std_ext::cap_std;
 use cap_std_ext::prelude::CapStdExtDirExt;
+use clap::ValueEnum;
 use rustix::fs::MetadataExt;
 
 use fn_error_context::context;
@@ -44,6 +45,7 @@ const BOOT: &str = "boot";
 const RUN_BOOTC: &str = "/run/bootc";
 /// This is an ext4 special directory we need to ignore.
 const LOST_AND_FOUND: &str = "lost+found";
+pub(crate) const ARCH_USES_EFI: bool = cfg!(any(target_arch = "x86_64", target_arch = "aarch64"));
 
 /// Kernel argument used to specify we want the rootfs mounted read-write by default
 const RW_KARG: &str = "rw";
@@ -117,6 +119,28 @@ pub(crate) struct InstallOpts {
     pub(crate) config_opts: InstallConfigOpts,
 }
 
+#[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) enum ReplaceMode {
+    /// Completely wipe the contents of the target filesystem.  This cannot
+    /// be done if the target filesystem is the one the system is booted from.
+    Wipe,
+    /// This is a destructive operation in the sense that the bootloader state
+    /// will have its contents wiped and replaced.  However,
+    /// the running system (and all files) will remain in place until reboot.
+    ///
+    /// As a corollary to this, you will also need to remove all the old operating
+    /// system binaries after the reboot into the target system; this can be done
+    /// with code in the new target system, or manually.
+    Alongside,
+}
+
+impl std::fmt::Display for ReplaceMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.to_possible_value().unwrap().get_name().fmt(f)
+    }
+}
+
 /// Options for installing to a filesystem
 #[derive(Debug, Clone, clap::Args)]
 pub(crate) struct InstallTargetFilesystemOpts {
@@ -141,9 +165,10 @@ pub(crate) struct InstallTargetFilesystemOpts {
     #[clap(long)]
     pub(crate) boot_mount_spec: Option<String>,
 
-    /// Automatically wipe existing data on the filesystems.
+    /// Initialize the system in-place; at the moment, only one mode for this is implemented.
+    /// In the future, it may also be supported to set up an explicit "dual boot" system.
     #[clap(long)]
-    pub(crate) wipe: bool,
+    pub(crate) replace: Option<ReplaceMode>,
 }
 
 /// Perform an installation to a mounted filesystem.
@@ -592,6 +617,8 @@ pub(crate) struct RootSetup {
     device: Utf8PathBuf,
     rootfs: Utf8PathBuf,
     rootfs_fd: Dir,
+    /// If true, do not try to remount the root read-only and flush the journal, etc.
+    skip_finalize: bool,
     boot: MountSpec,
     kargs: Vec<String>,
 }
@@ -826,9 +853,11 @@ async fn install_to_filesystem_impl(state: &State, rootfs: &mut RootSetup) -> Re
         .run()?;
 
     // Finalize mounted filesystems
-    let bootfs = rootfs.rootfs.join("boot");
-    for fs in [bootfs.as_path(), rootfs.rootfs.as_path()] {
-        finalize_filesystem(fs)?;
+    if !rootfs.skip_finalize {
+        let bootfs = rootfs.rootfs.join("boot");
+        for fs in [bootfs.as_path(), rootfs.rootfs.as_path()] {
+            finalize_filesystem(fs)?;
+        }
     }
 
     Ok(())
@@ -900,6 +929,34 @@ fn require_empty_rootdir(rootfs_fd: &Dir) -> Result<()> {
     Ok(())
 }
 
+/// Remove all entries in a directory, but do not traverse across distinct devices.
+#[context("Removing entries (noxdev")]
+fn remove_all_in_dir_no_xdev(d: &Dir) -> Result<()> {
+    let parent_dev = d.dir_metadata()?.dev();
+    for entry in d.entries()? {
+        let entry = entry?;
+        let entry_dev = entry.metadata()?.dev();
+        if entry_dev == parent_dev {
+            d.remove_all_optional(entry.file_name())?;
+        }
+    }
+    anyhow::Ok(())
+}
+
+#[context("Removing boot directory content")]
+fn clean_boot_directories(rootfs: &Dir) -> Result<()> {
+    let bootdir = rootfs.open_dir(BOOT).context("Opening /boot")?;
+    // This should not remove /boot/efi note.
+    remove_all_in_dir_no_xdev(&bootdir)?;
+    if ARCH_USES_EFI {
+        let efidir = bootdir
+            .open_dir(crate::bootloader::EFI_DIR)
+            .context("Opening /boot/efi")?;
+        remove_all_in_dir_no_xdev(&efidir)?;
+    }
+    Ok(())
+}
+
 /// Implementation of the `bootc install-to-filsystem` CLI command.
 pub(crate) async fn install_to_filesystem(opts: InstallToFilesystemOpts) -> Result<()> {
     // Gather global state, destructuring the provided options
@@ -909,19 +966,21 @@ pub(crate) async fn install_to_filesystem(opts: InstallToFilesystemOpts) -> Resu
     let root_path = &fsopts.root_path;
     let rootfs_fd = Dir::open_ambient_dir(root_path, cap_std::ambient_authority())
         .with_context(|| format!("Opening target root directory {root_path}"))?;
-    if fsopts.wipe {
-        let rootfs_fd = rootfs_fd.try_clone()?;
-        println!("Wiping contents of root");
-        tokio::task::spawn_blocking(move || {
-            for e in rootfs_fd.entries()? {
-                let e = e?;
-                rootfs_fd.remove_all_optional(e.file_name())?;
-            }
-            anyhow::Ok(())
-        })
-        .await??;
-    } else {
-        require_empty_rootdir(&rootfs_fd)?;
+    match fsopts.replace {
+        Some(ReplaceMode::Wipe) => {
+            let rootfs_fd = rootfs_fd.try_clone()?;
+            println!("Wiping contents of root");
+            tokio::task::spawn_blocking(move || {
+                for e in rootfs_fd.entries()? {
+                    let e = e?;
+                    rootfs_fd.remove_all_optional(e.file_name())?;
+                }
+                anyhow::Ok(())
+            })
+            .await??;
+        }
+        Some(ReplaceMode::Alongside) => clean_boot_directories(&rootfs_fd)?,
+        None => require_empty_rootdir(&rootfs_fd)?,
     }
 
     // Gather data about the root filesystem
@@ -1013,6 +1072,7 @@ pub(crate) async fn install_to_filesystem(opts: InstallToFilesystemOpts) -> Resu
         rootfs_fd,
         boot,
         kargs,
+        skip_finalize: matches!(fsopts.replace, Some(ReplaceMode::Alongside)),
     };
 
     install_to_filesystem_impl(&state, &mut rootfs).await?;

lib/src/install/baseline.rs

@@ -233,7 +233,7 @@ pub(crate) fn install_create_rootfs(
         anyhow::bail!("Unsupported architecture: {}", std::env::consts::ARCH);
     }
 
-    let espdev = if cfg!(any(target_arch = "x86_64", target_arch = "aarch64")) {
+    let espdev = if super::ARCH_USES_EFI {
         sgdisk_partition(
             &mut sgdisk.cmd,
             EFIPN,
@@ -370,5 +370,6 @@ pub(crate) fn install_create_rootfs(
         rootfs_fd,
         boot,
         kargs,
+        skip_finalize: false,
     })
 }

lib/src/mount.rs

@@ -13,6 +13,7 @@ use crate::task::Task;
 #[serde(rename_all = "kebab-case")]
 pub(crate) struct Filesystem {
     pub(crate) source: String,
+    pub(crate) sources: Option<Vec<String>>,
     pub(crate) uuid: Option<String>,
 }