container/commit: Auto-clean /var/tmp, /tmp, /run

The original command here was just scoped to `/var`, but we also
don't want content in `/run`.  Extend the tooling to handle that
and the other two temporary directories.

Also, let's be a bit nicer here and auto-clean empty directories
in `/var`.

I was testing out the
https://github.com/coreos/coreos-layering-examples/blob/main/tailscale/Dockerfile
example and today we have this:

```
drwxr-xr-x root/root         0 2022-09-13 18:53 run/
drwxr-xr-x root/root         0 2022-09-13 18:51 run/rpm-ostree/
drwxr-xr-x root/root         0 2022-09-13 18:53 run/rpm-ostree/lock/
drwxr-xr-x root/root         0 2022-09-13 18:51 run/systemd/
drwxr-xr-x root/root         0 2022-09-13 18:51 run/systemd/resolve/
-rwx------ root/root         0 2022-09-13 18:51 run/systemd/resolve/stub-resolv.conf
...
drwxr-xr-x root/root         0 2022-09-13 18:53 var/
drwxr-xr-x root/root         0 2022-09-13 18:53 var/cache/
drwx------ root/root         0 2022-09-13 18:53 var/cache/ldconfig/
-rw------- root/root     22000 2022-09-13 18:53 var/cache/ldconfig/aux-cache
drwxr-xr-x root/root         0 2022-09-08 23:10 var/cache/tailscale/
drwxr-xr-x root/root         0 2022-09-13 18:53 var/tmp/
```

In this set, we can auto-clean the leftover locking directories
rpm-ostree (erroneously) leaves in `/run`.

I am tempted to auto-cleanup the `stub-resolv.conf` thing and the
`ldconfig/aux-cache`.

lib/src/bootabletree.rs

@@ -43,7 +43,7 @@ pub fn find_kernel_dir(
     Ok(r)
 }
 
-fn read_dir_optional(
+pub(crate) fn read_dir_optional(
     d: &Dir,
     p: impl AsRef<Path>,
 ) -> std::io::Result<Option<cap_std::fs::ReadDir>> {

lib/src/commit.rs

@@ -7,14 +7,19 @@ use anyhow::Context;
 use anyhow::Result;
 use camino::Utf8Path;
 use camino::Utf8PathBuf;
+use cap_std::fs::Dir;
+use cap_std_ext::cap_std;
+use cap_std_ext::dirext::CapStdExtDirExt;
 use std::fs;
 use tokio::task;
 
-/// Check if there are any files that are not directories and error out if
-/// we find any, /var should not contain any files to commit in a container
-/// as it is where we expect user data to reside.
-fn validate_directories_only_recurse(path: &Utf8Path, error_count: &mut i32) -> Result<()> {
-    let context = || format!("Validating file: {:?}", path);
+/// Directories for which we will always remove all content.
+const FORCE_CLEAN_PATHS: &[&str] = &["run", "tmp", "var/tmp"];
+
+/// Gather count of non-empty directories.  Empty directories are removed.
+fn process_dir_recurse(path: &Utf8Path, error_count: &mut i32) -> Result<bool> {
+    let context = || format!("Validating: {path}");
+    let mut validated = true;
     for entry in fs::read_dir(path).with_context(context)? {
         let entry = entry?;
         let path = entry.path();
@@ -23,30 +28,45 @@ fn validate_directories_only_recurse(path: &Utf8Path, error_count: &mut i32) ->
         let metadata = path.symlink_metadata()?;
 
         if metadata.is_dir() {
-            validate_directories_only_recurse(&path, error_count)?;
+            if !process_dir_recurse(&path, error_count)? {
+                validated = false;
+            }
         } else {
+            validated = false;
             *error_count += 1;
             if *error_count < 20 {
                 eprintln!("Found file: {:?}", path)
             }
         }
     }
-    Ok(())
+    if validated {
+        fs::remove_dir(path).with_context(context)?;
+    }
+    Ok(validated)
 }
 
-fn validate_ostree_compatibility_in(root: &Utf8Path) -> Result<()> {
-    let var_path = root.join("var");
-    println!("Checking /var for files");
+/// Given a root filesystem, clean out empty directories and warn about
+/// files in /var.  /run, /tmp, and /var/tmp have their contents recursively cleaned.
+fn prepare_ostree_commit_in(root: &Utf8Path) -> Result<()> {
     let mut error_count = 0;
-    validate_directories_only_recurse(&var_path, &mut error_count)?;
-    if error_count != 0 {
-        anyhow::bail!("Found content in {var_path}");
+    let rootdir = Dir::open_ambient_dir(root, cap_std::ambient_authority())?;
+    for path in FORCE_CLEAN_PATHS {
+        if let Some(subdir) = rootdir.open_dir_optional(path)? {
+            for entry in subdir.entries()? {
+                let entry = entry?;
+                subdir.remove_all_optional(entry.file_name())?;
+            }
+        }
+    }
+    let var = root.join("var");
+    if var.exists() && !process_dir_recurse(&var, &mut error_count)? {
+        anyhow::bail!("Found content in {var}");
     }
     Ok(())
 }
 
 fn validate_ostree_compatibility() -> Result<()> {
-    validate_ostree_compatibility_in(Utf8Path::new("/"))
+    prepare_ostree_commit_in(Utf8Path::new("/"))
 }
 
 /// Entrypoint to the commit procedures, initially we just
@@ -67,14 +87,46 @@ mod tests {
         let td = td.path();
         let td = Utf8Path::from_path(td).unwrap();
 
-        let var = td.join("var");
+        // Handle the empty case
+        prepare_ostree_commit_in(td).unwrap();
 
-        std::fs::create_dir(&var)?;
-        validate_ostree_compatibility_in(td).unwrap();
+        let var = &td.join("var");
+        let run = &td.join("run");
+        let tmp = &td.join("tmp");
+        let vartmp_foobar = &var.join("tmp/foo/bar");
+        let runsystemd = &run.join("systemd");
+        let resolvstub = &runsystemd.join("resolv.conf");
+
+        for p in [var, run, tmp] {
+            std::fs::create_dir(p)?;
+        }
 
+        std::fs::create_dir_all(vartmp_foobar)?;
+        std::fs::write(vartmp_foobar.join("a"), "somefile")?;
+        std::fs::write(vartmp_foobar.join("b"), "somefile2")?;
+        std::fs::create_dir_all(runsystemd)?;
+        std::fs::write(resolvstub, "stub resolv")?;
+        prepare_ostree_commit_in(td).unwrap();
+        assert!(!var.exists());
+        assert!(run.exists());
+        assert!(!runsystemd.exists());
+
+        let systemd = run.join("systemd");
+        std::fs::create_dir_all(&systemd)?;
+        prepare_ostree_commit_in(td).unwrap();
+        assert!(!var.exists());
+
+        std::fs::create_dir(&var)?;
         std::fs::write(var.join("foo"), "somefile")?;
+        assert!(prepare_ostree_commit_in(td).is_err());
+        assert!(var.exists());
 
-        assert!(validate_ostree_compatibility_in(td).is_err());
+        let nested = td.join("var/lib/nested");
+        std::fs::create_dir_all(&nested)?;
+        std::fs::write(nested.join("foo"), "test1")?;
+        std::fs::write(nested.join("foo2"), "test2")?;
+        assert!(prepare_ostree_commit_in(td).is_err());
+        assert!(var.exists());
 
         Ok(())
     }