Normalise various aspects of the RPM database that vary build-to-build

Cargo.toml

@@ -29,6 +29,7 @@ rpm = "4"
 
 [dependencies]
 anyhow = "1.0.44"
+binread = "2.2.0"
 c_utf8 = "0.1.0"
 camino = "1.0.5"
 chrono = { version = "0.4.19", features = ["serde"] }
@@ -50,6 +51,7 @@ memfd = "0.4.1"
 nix = "0.23.0"
 openat = "0.1.21"
 openat-ext = "^0.2.2"
+openssl = "0.10.38"
 os-release = "0.1.0"
 ostree-ext = "0.3.0"
 paste = "1.0"

docs/treefile.md

@@ -111,6 +111,13 @@ It supports the following parameters:
     library in the target filesystem tree understands.  However, this is
     a relatively new default, so the value `host` is provided as a fallback
 
+ * `rpmdb-normalize`: boolean, optional. Defaults to `false`.  If enabled,
+    this will perform various manipulations of the RPM database to, as much
+    as possible, guarantee a deterministic result for the on-disk RPM
+    database. Requires the `SOURCE_DATE_EPOCH` environment variable to be set
+    to the UNIX epoch time to be used as the build timestamp. Currently only
+    fully supports the `bdb` backend. Somewhat experimental.
+
  * `cliwrap`: boolean, optional.  Defaults to `false`.  If enabled,
     rpm-ostree will replace binaries such as `/usr/bin/rpm` with
     wrappers that intercept unsafe operations, or adjust functionality.

rust/src/composepost.rs

@@ -9,6 +9,7 @@ use crate::bwrap::Bubblewrap;
 use crate::cxxrsutil::*;
 use crate::ffi::BubblewrapMutability;
 use crate::ffiutil::ffi_view_openat_dir;
+use crate::normalization;
 use crate::passwd::PasswdDB;
 use crate::treefile::Treefile;
 use crate::{bwrap, importer};
@@ -951,7 +952,7 @@ fn hardlink_rpmdb_base_location(
 }
 
 #[context("Rewriting rpmdb for target native format")]
-fn rewrite_rpmdb_for_target_inner(rootfs_dfd: &openat::Dir) -> Result<()> {
+fn rewrite_rpmdb_for_target_inner(rootfs_dfd: &openat::Dir, normalize: bool) -> Result<()> {
     let tempetc = crate::core::prepare_tempetc_guard(rootfs_dfd.as_raw_fd())?;
 
     let dbfd = Rc::new(
@@ -980,6 +981,12 @@ fn rewrite_rpmdb_for_target_inner(rootfs_dfd: &openat::Dir) -> Result<()> {
     let mut dbfd = Rc::try_unwrap(dbfd).unwrap();
     dbfd.seek(std::io::SeekFrom::Start(0))?;
 
+    // In the interests of build stability, rewrite the INSTALLTIME and INSTALLTID tags
+    // to be deterministic and dervied from `SOURCE_DATE_EPOCH` if requested.
+    if normalize {
+        normalization::rewrite_rpmdb_timestamps(&mut dbfd)?;
+    }
+
     // Fork the target rpmdb to write the content from memory to disk
     let mut bwrap = Bubblewrap::new_with_mutability(rootfs_dfd, BubblewrapMutability::RoFiles)?;
     bwrap.append_child_argv(&["rpmdb", dbpath_arg.as_str(), "--importdb"]);
@@ -989,15 +996,22 @@ fn rewrite_rpmdb_for_target_inner(rootfs_dfd: &openat::Dir) -> Result<()> {
         .run(cancellable.gobj_rewrap())
         .context("Failed to run rpmdb --importdb")?;
 
+    // Sometimes we can end up with build-to-build variance in the underlying rpmdb
+    // files. Attempt to sort that out, if requested.
+    if normalize {
+        normalization::normalize_rpmdb(rootfs_dfd, RPMOSTREE_RPMDB_LOCATION)?;
+    }
+
     tempetc.undo()?;
 
     Ok(())
 }
 
-pub(crate) fn rewrite_rpmdb_for_target(rootfs_dfd: i32) -> CxxResult<()> {
-    Ok(rewrite_rpmdb_for_target_inner(&ffi_view_openat_dir(
-        rootfs_dfd,
-    ))?)
+pub(crate) fn rewrite_rpmdb_for_target(rootfs_dfd: i32, normalize: bool) -> CxxResult<()> {
+    Ok(rewrite_rpmdb_for_target_inner(
+        &ffi_view_openat_dir(rootfs_dfd),
+        normalize,
+    )?)
 }
 
 /// Recursively hard-link `source` hierarchy to `target` directory.

rust/src/lib.rs

@@ -199,7 +199,7 @@ pub mod ffi {
             cancellable: Pin<&mut GCancellable>,
         ) -> Result<()>;
         fn compose_postprocess_rpm_macro(rootfs_dfd: i32) -> Result<()>;
-        fn rewrite_rpmdb_for_target(rootfs_dfd: i32) -> Result<()>;
+        fn rewrite_rpmdb_for_target(rootfs_dfd: i32, normalize: bool) -> Result<()>;
         fn directory_size(dfd: i32, mut cancellable: Pin<&mut GCancellable>) -> Result<u64>;
     }
 
@@ -370,6 +370,7 @@ pub mod ffi {
         fn get_selinux(&self) -> bool;
         fn get_releasever(&self) -> &str;
         fn rpmdb_backend_is_target(&self) -> bool;
+        fn should_normalize_rpmdb(&self) -> bool;
         fn get_files_remove_regex(&self, package: &str) -> Vec<String>;
         fn get_checksum(&self, repo: Pin<&mut OstreeRepo>) -> Result<String>;
         fn get_ostree_ref(&self) -> String;