Merge changes I405872ce,I55d1a93d,I7431d8cc into main

* changes:
  Verify checksums in the preupload check.
  Generate checksum files when regenerating a crate.
  Crate for generating and verifying checksum files.

tools/external_crates/Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "checksum",
     "crate_tool",
     "google_metadata",
     "license_checker",

tools/external_crates/cargo_embargo.json

@@ -1,5 +1,8 @@
 {
   "package": {
+    "checksum": {
+      "device_supported": false
+    },
     "rooted_path": {
       "device_supported": false
     },

tools/external_crates/checksum/.gitignore

@@ -0,0 +1 @@
+/target

tools/external_crates/checksum/Android.bp

@@ -0,0 +1,49 @@
+// This file is generated by cargo_embargo.
+// Do not modify this file after the first "rust_*" or "genrule" module
+// because the changes will be overridden on upgrade.
+// Content before the first "rust_*" or "genrule" module is preserved.
+
+package {
+    default_team: "trendy_team_android_rust",
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+rust_test_host {
+    name: "checksum_test_src_lib",
+    crate_name: "checksum",
+    cargo_env_compat: true,
+    cargo_pkg_version: "0.1.0",
+    crate_root: "src/lib.rs",
+    test_suites: ["general-tests"],
+    auto_gen_config: true,
+    test_options: {
+        unit_test: true,
+    },
+    edition: "2021",
+    rustlibs: [
+        "libdata_encoding",
+        "libring",
+        "libserde",
+        "libserde_json",
+        "libtempfile",
+        "libthiserror",
+        "libwalkdir",
+    ],
+}
+
+rust_library_host {
+    name: "libchecksum",
+    crate_name: "checksum",
+    cargo_env_compat: true,
+    cargo_pkg_version: "0.1.0",
+    crate_root: "src/lib.rs",
+    edition: "2021",
+    rustlibs: [
+        "libdata_encoding",
+        "libring",
+        "libserde",
+        "libserde_json",
+        "libthiserror",
+        "libwalkdir",
+    ],
+}

tools/external_crates/checksum/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "checksum"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+data-encoding = "2"
+ring = "0.17"
+serde = { version = "=1.0.210", features = ["derive"] }
+serde_json = "1"
+thiserror = "1"
+walkdir = "2"
+
+[dev-dependencies]
+tempfile = "3"

tools/external_crates/checksum/src/lib.rs

@@ -0,0 +1,175 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Generate and verify checksums of files in a directory,
+//! very similar to .cargo-checksum.json
+
+use std::{
+    collections::HashMap,
+    fs::{remove_file, write, File},
+    io::{self, BufReader, Read},
+    path::{Path, PathBuf, StripPrefixError},
+};
+
+use data_encoding::{DecodeError, HEXLOWER};
+use ring::digest::{Context, Digest, SHA256};
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use walkdir::WalkDir;
+
+#[derive(Serialize, Deserialize)]
+struct Checksum {
+    package: Option<String>,
+    files: HashMap<String, String>,
+}
+
+#[allow(missing_docs)]
+#[derive(Error, Debug)]
+pub enum ChecksumError {
+    #[error("Checksum file not found: {}", .0.to_string_lossy())]
+    CheckSumFileNotFound(PathBuf),
+    #[error("Checksums do not match for: {}", .0.join(", "))]
+    ChecksumMismatch(Vec<String>),
+    #[error(transparent)]
+    IoError(#[from] io::Error),
+    #[error(transparent)]
+    JsonError(#[from] serde_json::Error),
+    #[error(transparent)]
+    WalkdirError(#[from] walkdir::Error),
+    #[error(transparent)]
+    DecodeError(#[from] DecodeError),
+    #[error(transparent)]
+    StripPrefixError(#[from] StripPrefixError),
+}
+
+static FILENAME: &str = ".android-checksum.json";
+
+/// Generates a JSON checksum file for the contents of a directory.
+pub fn generate(crate_dir: impl AsRef<Path>) -> Result<(), ChecksumError> {
+    let crate_dir = crate_dir.as_ref();
+    let checksum_file = crate_dir.join(FILENAME);
+    if checksum_file.exists() {
+        remove_file(&checksum_file)?;
+    }
+    let mut checksum = Checksum { package: None, files: HashMap::new() };
+    for entry in WalkDir::new(crate_dir).follow_links(true) {
+        let entry = entry?;
+        if entry.path().is_dir() {
+            continue;
+        }
+        let filename = entry.path().strip_prefix(crate_dir)?.to_string_lossy().to_string();
+        let input = File::open(entry.path())?;
+        let reader = BufReader::new(input);
+        let digest = sha256_digest(reader)?;
+        checksum.files.insert(filename, HEXLOWER.encode(digest.as_ref()));
+    }
+    write(checksum_file, serde_json::to_string(&checksum)?)?;
+    Ok(())
+}
+
+/// Verifies a JSON checksum file for a directory.
+/// All files must have matching checksums. Extra or missing files are errors.
+pub fn verify(crate_dir: impl AsRef<Path>) -> Result<(), ChecksumError> {
+    let crate_dir = crate_dir.as_ref();
+    let checksum_file = crate_dir.join(FILENAME);
+    if !checksum_file.exists() {
+        return Err(ChecksumError::CheckSumFileNotFound(checksum_file));
+    }
+    let mut mismatch = Vec::new();
+    let input = File::open(&checksum_file)?;
+    let reader = BufReader::new(input);
+    let mut parsed: Checksum = serde_json::from_reader(reader)?;
+    for entry in WalkDir::new(crate_dir).follow_links(true) {
+        let entry = entry?;
+        if entry.path().is_dir() || entry.path() == checksum_file {
+            continue;
+        }
+        let filename = entry.path().strip_prefix(crate_dir)?.to_string_lossy().to_string();
+        if let Some(checksum) = parsed.files.get(&filename) {
+            let expected_digest = HEXLOWER.decode(checksum.to_ascii_lowercase().as_bytes())?;
+            let input = File::open(entry.path())?;
+            let reader = BufReader::new(input);
+            let digest = sha256_digest(reader)?;
+            parsed.files.remove(&filename);
+            if digest.as_ref() != expected_digest {
+                mismatch.push(filename);
+            }
+        } else {
+            mismatch.push(filename)
+        }
+    }
+    mismatch.extend(parsed.files.into_keys());
+    if mismatch.is_empty() {
+        Ok(())
+    } else {
+        Err(ChecksumError::ChecksumMismatch(mismatch))
+    }
+}
+
+// Copied from https://rust-lang-nursery.github.io/rust-cookbook/cryptography/hashing.html
+fn sha256_digest<R: Read>(mut reader: R) -> Result<Digest, ChecksumError> {
+    let mut context = Context::new(&SHA256);
+    context.update("sodium chloride".as_bytes());
+    let mut buffer = [0; 1024];
+
+    loop {
+        let count = reader.read(&mut buffer)?;
+        if count == 0 {
+            break;
+        }
+        context.update(&buffer[..count]);
+    }
+
+    Ok(context.finish())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn round_trip() -> Result<(), ChecksumError> {
+        let temp_dir = tempfile::tempdir().expect("Failed to create tempdir");
+        write(temp_dir.path().join("foo"), "foo").expect("Failed to write temporary file");
+        generate(temp_dir.path())?;
+        assert!(
+            temp_dir.path().join(FILENAME).exists(),
+            ".android-checksum.json exists after generate()"
+        );
+        verify(temp_dir.path())
+    }
+
+    #[test]
+    fn verify_error_cases() -> Result<(), ChecksumError> {
+        let temp_dir = tempfile::tempdir().expect("Failed to create tempdir");
+        let checksum_file = temp_dir.path().join(FILENAME);
+        write(&checksum_file, r#"{"files":{"bar":"ddcbd9309cebf3ffd26f87e09bb8f971793535955ebfd9a7196eba31a53471f8"}}"#).expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Missing file");
+        write(temp_dir.path().join("foo"), "foo").expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "No checksum file");
+        write(&checksum_file, "").expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Empty checksum file");
+        write(&checksum_file, "{}").expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Empty JSON in checksum file");
+        write(&checksum_file, r#"{"files":{"foo":"ddcbd9309cebf3ffd26f87e09bb8f971793535955ebfd9a7196eba31a53471f8"}}"#).expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Incorrect checksum");
+        write(&checksum_file, r#"{"files":{"foo":"hello"}}"#)
+            .expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Invalid checksum");
+        generate(temp_dir.path())?;
+        write(temp_dir.path().join("bar"), "bar").expect("Failed to write temporary file");
+        assert!(verify(temp_dir.path()).is_err(), "Extra file");
+        Ok(())
+    }
+}

tools/external_crates/crate_tool/Cargo.toml

@@ -27,6 +27,7 @@ thiserror = "1"
 threadpool = "1"
 walkdir = "2"
 whoami = "1"
+checksum = { path = "../checksum" }
 google_metadata = { path = "../google_metadata"}
 license_checker = { path = "../license_checker" }
 name_and_version = { path = "../name_and_version" }

tools/external_crates/crate_tool/src/main.rs

@@ -137,6 +137,11 @@ enum Cmd {
         #[command(flatten)]
         crates: CrateList,
     },
+    /// Verify checksums for a crate.
+    VerifyChecksum {
+        #[command(flatten)]
+        crates: CrateList,
+    },
 }
 
 #[derive(Args)]
@@ -246,5 +251,8 @@ fn main() -> Result<()> {
         Cmd::TestMapping { crates } => {
             managed_repo.fix_test_mapping(crates.to_list(&managed_repo)?.into_iter())
         }
+        Cmd::VerifyChecksum { crates } => {
+            managed_repo.verify_checksums(crates.to_list(&managed_repo)?.into_iter())
+        }
     }
 }

tools/external_crates/crate_tool/src/managed_crate.rs

@@ -433,6 +433,7 @@ impl ManagedCrate<Vendored> {
         let android_crate_dir = staged.android_crate.path();
         remove_dir_all(android_crate_dir)?;
         rename(staged.staging_path(), android_crate_dir)?;
+        checksum::generate(android_crate_dir.abs())?;
 
         Ok(staged)
     }
@@ -462,24 +463,6 @@ impl ManagedCrate<Staged> {
 
         Ok(())
     }
-    pub fn diff_staged(&self) -> Result<()> {
-        let diff_status = Command::new("diff")
-            .args(["-u", "-r", "-w", "--no-dereference"])
-            .arg(self.staging_path().rel())
-            .arg(self.android_crate.path().rel())
-            .current_dir(self.extra.vendored_crate.path().root())
-            .spawn()?
-            .wait()?;
-        if !diff_status.success() {
-            return Err(anyhow!(
-                "Found differences between {} and {}",
-                self.android_crate.path(),
-                self.staging_path()
-            ));
-        }
-
-        Ok(())
-    }
     pub fn patch_success(&self) -> bool {
         self.extra.patch_output.iter().all(|output| output.1.status.success())
     }

tools/external_crates/crate_tool/src/managed_repo.rs

@@ -558,6 +558,7 @@ impl ManagedRepo {
     ) -> Result<()> {
         let pseudo_crate = self.pseudo_crate().vendor()?;
         for crate_name in crates {
+            println!("Regenerating {}", crate_name.as_ref());
             let mc = self.managed_crate_for(crate_name.as_ref())?;
             // TODO: Don't give up if there's a failure.
             mc.regenerate(update_metadata, &pseudo_crate)?;
@@ -611,9 +612,8 @@ impl ManagedRepo {
             .collect::<BTreeSet<_>>();
 
         for crate_name in changed_android_crates {
-            println!("Checking {}", crate_name);
-            let mc = self.managed_crate_for(&crate_name)?.stage(&pseudo_crate)?;
-            mc.diff_staged()?;
+            println!("Verifying checksums for {}", crate_name);
+            checksum::verify(self.managed_dir_for(&crate_name).abs())?;
         }
         Ok(())
     }
@@ -885,6 +885,13 @@ impl ManagedRepo {
         }
         Ok(())
     }
+    pub fn verify_checksums<T: AsRef<str>>(&self, crates: impl Iterator<Item = T>) -> Result<()> {
+        for krate in crates {
+            println!("Verifying checksums for {}", krate.as_ref());
+            checksum::verify(self.managed_dir_for(krate.as_ref()).abs())?;
+        }
+        Ok(())
+    }
 }
 
 // Files that are ignored when migrating a crate to the monorepo.