feat: add API to generate tree-r-last and tree-c (#1705)

It's now possible to create the merkle tree on top of a replica by
specifying the replica and the output directory.

It's also possible to create the merkle tree on top of labels by
specifying the path to the labels and the output directory.

fil-proofs-tooling/Cargo.toml

@@ -29,7 +29,7 @@ cpu-time = "1.0.0"
 blake2s_simd = "1.0.0"
 fil_logger = "0.1.6"
 log = "0.4.8"
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 bincode = "1.1.2"
 anyhow = "1.0.23"
 rand_xorshift = "0.3.0"

filecoin-hashers/Cargo.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 bellperson = "0.25.0"
 blstrs = "0.7.0"
 generic-array = "0.14.4"
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 ff = "0.13.0"
 anyhow = "1.0.34"
 serde = "1.0.117"

filecoin-proofs/Cargo.toml

@@ -24,7 +24,7 @@ bellperson = "0.25.0"
 log = "0.4.7"
 rayon = "1.1.0"
 hex = "0.4.0"
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 bincode = "1.1.2"
 anyhow = "1.0.23"
 sha2 = "0.10.2"

filecoin-proofs/src/api/seal.rs

@@ -2,7 +2,7 @@ use std::fs::{self, metadata, File, OpenOptions};
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
-use anyhow::{ensure, Context, Result};
+use anyhow::{anyhow, ensure, Context, Result};
 use bellperson::groth16;
 use bincode::{deserialize, serialize};
 use blstrs::{Bls12, Scalar as Fr};
@@ -17,18 +17,23 @@ use storage_proofs_core::{
     compound_proof::{self, CompoundProof},
     drgraph::Graph,
     measurements::{measure_op, Operation},
-    merkle::{create_base_merkle_tree, BinaryMerkleTree, MerkleTreeTrait},
+    merkle::{
+        create_base_merkle_tree, get_base_tree_count, split_config, BinaryMerkleTree,
+        MerkleTreeTrait,
+    },
     multi_proof::MultiProof,
     parameter_cache::SRS_MAX_PROOFS_TO_AGGREGATE,
     proof::ProofScheme,
     sector::SectorId,
-    util::default_rows_to_discard,
+    util::{default_rows_to_discard, NODE_SIZE},
     Data,
 };
 use storage_proofs_porep::stacked::{
-    self, generate_replica_id, ChallengeRequirements, StackedCompound, StackedDrg, Tau,
-    TemporaryAux, TemporaryAuxCache,
+    self, generate_replica_id, ChallengeRequirements, Labels, LabelsCache, StackedCompound,
+    StackedDrg, Tau, TemporaryAux, TemporaryAuxCache,
 };
+use storage_proofs_update::vanilla::prepare_tree_r_data;
+use typenum::{Unsigned, U11, U2};
 
 use crate::POREP_MINIMUM_CHALLENGES;
 use crate::{
@@ -1130,3 +1135,119 @@ pub fn verify_batch_seal<Tree: 'static + MerkleTreeTrait>(
     info!("verify_batch_seal:finish");
     result
 }
+
+/// Generate the merkle tree on top of the replica (TreeRLast).
+///
+/// The generated trees are stored in `output_dir`, usually the cache directory. The `replica_path`
+/// point to the replica where the tree should be built upon. The `sector_size` is in bytes.
+pub fn generate_tree_r_last<O, R, TreeR: 'static + MerkleTreeTrait>(
+    sector_size: u64,
+    replica_path: R,
+    output_dir: O,
+) -> Result<<TreeR::Hasher as Hasher>::Domain>
+where
+    O: AsRef<Path>,
+    R: AsRef<Path>,
+{
+    let leaf_count = sector_size as usize / NODE_SIZE;
+    let base_tree_count = get_base_tree_count::<TreeR>();
+    let base_tree_leafs = leaf_count / base_tree_count;
+
+    let rows_to_discard = default_rows_to_discard(base_tree_leafs, TreeR::Arity::to_usize());
+    let size = get_base_tree_size::<TreeR>(SectorSize(sector_size))?;
+    let tree_r_last_config = StoreConfig {
+        path: PathBuf::from(output_dir.as_ref()),
+        id: CacheKey::CommRLastTree.to_string(),
+        size: Some(size),
+        // A default 'rows_to_discard' value will be chosen for tree_r_last, unless the user
+        // overrides this value via the environment setting (FIL_PROOFS_ROWS_TO_DISCARD). If
+        // this value is specified, no checking is done on it and it may result in a broken
+        // configuration. *Use with caution*. It must be noted that if/when this unchecked
+        // value is passed through merkle_light, merkle_light now does a check that does not
+        // allow us to discard more rows than is possible to discard.
+        rows_to_discard,
+    };
+
+    let replica_base_tree_size = get_base_tree_size::<DefaultBinaryTree>(sector_size.into())?;
+    let replica_base_tree_leafs = get_base_tree_leafs::<DefaultBinaryTree>(replica_base_tree_size)?;
+    let replica = DiskStore::new_from_disk_with_path(replica_base_tree_leafs, &replica_path)?;
+
+    // This argument is currently unused by this invocation, but required for the API.
+    let mut unused_data = Data::empty();
+
+    let tree_r_last = StackedDrg::<TreeR, DefaultPieceHasher>::generate_tree_r_last(
+        &mut unused_data,
+        base_tree_leafs,
+        base_tree_count,
+        tree_r_last_config,
+        PathBuf::from(replica_path.as_ref()),
+        &replica,
+        // By default, the replica file is manipulated, use the prepare function from the empty
+        // sector update, that only prepares the data for use on the GPU if needed.
+        Some(prepare_tree_r_data),
+    )?;
+    Ok(tree_r_last.root())
+}
+
+/// Generate the merkle tree on top of the labels (TreeC).
+///
+/// The generated trees are stored in `output_dir`, usually the cache directory. The `input_dir`
+/// points to the directory where the labels are stored, usually the cache directory. The
+/// `sector_size` is in bytes.
+pub fn generate_tree_c<I, O, Tree: 'static + MerkleTreeTrait>(
+    sector_size: u64,
+    input_dir: I,
+    output_dir: O,
+    num_layers: usize,
+) -> Result<<Tree::Hasher as Hasher>::Domain>
+where
+    I: AsRef<Path>,
+    O: AsRef<Path>,
+{
+    let leaf_count = sector_size as usize / NODE_SIZE;
+    let base_tree_count = get_base_tree_count::<Tree>();
+    let base_tree_leafs = leaf_count / base_tree_count;
+
+    let rows_to_discard = default_rows_to_discard(base_tree_leafs, Tree::Arity::to_usize());
+    let size = get_base_tree_size::<Tree>(SectorSize(sector_size))?;
+    let tree_c_config = StoreConfig {
+        path: PathBuf::from(output_dir.as_ref()),
+        id: CacheKey::CommCTree.to_string(),
+        size: Some(size),
+        rows_to_discard,
+    };
+    let configs = split_config(tree_c_config, base_tree_count)?;
+
+    let labels_cache = {
+        let label_base_tree_size = get_base_tree_size::<DefaultBinaryTree>(sector_size.into())?;
+        let label_base_tree_leafs = get_base_tree_leafs::<DefaultBinaryTree>(label_base_tree_size)?;
+        let label_configs = (1..=num_layers)
+            .map(|layer| StoreConfig {
+                path: PathBuf::from(input_dir.as_ref()),
+                id: CacheKey::label_layer(layer),
+                size: Some(label_base_tree_leafs),
+                rows_to_discard: default_rows_to_discard(label_base_tree_leafs, BINARY_ARITY),
+            })
+            .collect();
+        let labels = Labels::new(label_configs);
+        LabelsCache::<Tree>::new(&labels).context("failed to create labels cache")?
+    };
+
+    let tree_c = match num_layers {
+        2 => StackedDrg::<Tree, DefaultPieceHasher>::generate_tree_c::<U2, Tree::Arity>(
+            base_tree_leafs,
+            base_tree_count,
+            configs,
+            &labels_cache,
+        )?,
+        11 => StackedDrg::<Tree, DefaultPieceHasher>::generate_tree_c::<U11, Tree::Arity>(
+            base_tree_leafs,
+            base_tree_count,
+            configs,
+            &labels_cache,
+        )?,
+        _ => return Err(anyhow!("Unsupported column arity")),
+    };
+
+    Ok(tree_c.root())
+}

filecoin-proofs/tests/api.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
-use std::fs::{metadata, read_dir, remove_file, OpenOptions};
-use std::io::{Read, Seek, SeekFrom, Write};
+use std::fs::{metadata, read_dir, remove_file, File, OpenOptions};
+use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context, Error, Result};
@@ -14,17 +14,17 @@ use filecoin_proofs::{
     decode_from_range, encode_into, fauxrep_aux, generate_empty_sector_update_proof,
     generate_empty_sector_update_proof_with_vanilla, generate_fallback_sector_challenges,
     generate_partition_proofs, generate_piece_commitment, generate_single_partition_proof,
-    generate_single_vanilla_proof, generate_single_window_post_with_vanilla, generate_window_post,
-    generate_window_post_with_vanilla, generate_winning_post,
-    generate_winning_post_sector_challenge, generate_winning_post_with_vanilla,
-    get_num_partition_for_fallback_post, get_seal_inputs, merge_window_post_partition_proofs,
-    remove_encoded_data, seal_commit_phase1, seal_commit_phase2, seal_pre_commit_phase1,
-    seal_pre_commit_phase2, unseal_range, validate_cache_for_commit,
-    validate_cache_for_precommit_phase2, verify_aggregate_seal_commit_proofs,
-    verify_empty_sector_update_proof, verify_partition_proofs, verify_seal,
-    verify_single_partition_proof, verify_window_post, verify_winning_post, Commitment,
-    DefaultTreeDomain, MerkleTreeTrait, PaddedBytesAmount, PieceInfo, PoRepConfig, PoStConfig,
-    PoStType, PrivateReplicaInfo, ProverId, PublicReplicaInfo, SealCommitOutput,
+    generate_single_vanilla_proof, generate_single_window_post_with_vanilla, generate_tree_c,
+    generate_tree_r_last, generate_window_post, generate_window_post_with_vanilla,
+    generate_winning_post, generate_winning_post_sector_challenge,
+    generate_winning_post_with_vanilla, get_num_partition_for_fallback_post, get_seal_inputs,
+    merge_window_post_partition_proofs, remove_encoded_data, seal_commit_phase1,
+    seal_commit_phase2, seal_pre_commit_phase1, seal_pre_commit_phase2, unseal_range,
+    validate_cache_for_commit, validate_cache_for_precommit_phase2,
+    verify_aggregate_seal_commit_proofs, verify_empty_sector_update_proof, verify_partition_proofs,
+    verify_seal, verify_single_partition_proof, verify_window_post, verify_winning_post,
+    Commitment, DefaultTreeDomain, MerkleTreeTrait, PaddedBytesAmount, PieceInfo, PoRepConfig,
+    PoStConfig, PoStType, PrivateReplicaInfo, ProverId, PublicReplicaInfo, SealCommitOutput,
     SealPreCommitOutput, SealPreCommitPhase1Output, SectorShape16KiB, SectorShape2KiB,
     SectorShape32KiB, SectorShape4KiB, SectorUpdateConfig, UnpaddedByteIndex, UnpaddedBytesAmount,
     SECTOR_SIZE_16_KIB, SECTOR_SIZE_2_KIB, SECTOR_SIZE_32_KIB, SECTOR_SIZE_4_KIB,
@@ -34,10 +34,13 @@ use filecoin_proofs::{
 use fr32::bytes_into_fr;
 use log::info;
 use memmap2::MmapOptions;
+use merkletree::store::StoreConfig;
 use rand::{random, Rng, SeedableRng};
 use rand_xorshift::XorShiftRng;
+use sha2::{Digest, Sha256};
 use storage_proofs_core::{
-    api_version::ApiVersion, is_legacy_porep_id, sector::SectorId, util::NODE_SIZE,
+    api_version::ApiVersion, cache_key::CacheKey, is_legacy_porep_id, merkle::get_base_tree_count,
+    sector::SectorId, util::NODE_SIZE,
 };
 use storage_proofs_update::constants::TreeRHasher;
 use tempfile::{tempdir, NamedTempFile, TempDir};
@@ -1716,13 +1719,26 @@ fn create_seal<R: Rng, Tree: 'static + MerkleTreeTrait>(
         &sealed_sector_file,
     )?;
 
+    let num_layers = phase1_output.labels.len();
     let pre_commit_output = seal_pre_commit_phase2(
         &config,
         phase1_output,
         cache_dir.path(),
         sealed_sector_file.path(),
     )?;
 
+    // Check if creating only the tree_r_last generates the same output as the full pre commit
+    // phase 2 process.
+    let tree_r_last_dir = tempdir().expect("failed to create temp dir");
+    generate_tree_r_last::<_, _, Tree>(sector_size, &sealed_sector_file, &tree_r_last_dir)?;
+    compare_trees::<Tree>(&tree_r_last_dir, &cache_dir, CacheKey::CommRLastTree)?;
+
+    // Check if creating only the tree_r generates the same output as the full pre commit phase 2
+    // process.
+    let tree_c_dir = tempdir().expect("failed to create temp dir");
+    generate_tree_c::<_, _, Tree>(sector_size, &cache_dir, &tree_c_dir, num_layers)?;
+    compare_trees::<Tree>(&tree_c_dir, &cache_dir, CacheKey::CommCTree)?;
+
     let comm_r = pre_commit_output.comm_r;
 
     validate_cache_for_commit::<_, _, Tree>(cache_dir.path(), sealed_sector_file.path())?;
@@ -1833,6 +1849,39 @@ fn compare_elements(path1: &Path, path2: &Path) -> Result<(), Error> {
     Ok(())
 }
 
+/// Return the hash of a given file specified by cache key within a certain directory.
+fn hash_file(dir: &TempDir, cache_key: &str) -> Result<Vec<u8>> {
+    let path = StoreConfig::data_path(dir.path(), cache_key);
+    let mut hasher = Sha256::new();
+    let mut file = File::open(path)?;
+    io::copy(&mut file, &mut hasher)?;
+    Ok(hasher.finalize().to_vec())
+}
+
+/// Compare whether two trees are identical.
+///
+/// The tree may be split across several files.
+fn compare_trees<Tree: 'static + MerkleTreeTrait>(
+    dir_a: &TempDir,
+    dir_b: &TempDir,
+    cache_key: CacheKey,
+) -> Result<()> {
+    let base_tree_count = get_base_tree_count::<Tree>();
+    let cache_key_names = if base_tree_count == 1 {
+        vec![cache_key.to_string()]
+    } else {
+        (0..base_tree_count)
+            .map(|count| format!("{}-{}", cache_key, count))
+            .collect()
+    };
+    for cache_key_name in cache_key_names {
+        let hash_a = hash_file(dir_a, &cache_key_name)?;
+        let hash_b = hash_file(dir_b, &cache_key_name)?;
+        assert_eq!(hash_a, hash_b, "files are identical");
+    }
+    Ok(())
+}
+
 /// Returns the decoded data.
 ///
 /// The decoding is done in several arbitrarily sized parts.

storage-proofs-core/Cargo.toml

@@ -14,7 +14,7 @@ bench = false
 [dependencies]
 filecoin-hashers = { path = "../filecoin-hashers", version = "~9.0.0", default-features = false, features = ["sha256", "poseidon"] }
 rand = "0.8"
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 byteorder = "1"
 config = { version = "0.12.0", default-features = false, features = ["toml"] }
 itertools = "0.10.3"

storage-proofs-porep/Cargo.toml

@@ -13,7 +13,7 @@ crossbeam = "0.8"
 storage-proofs-core = { path = "../storage-proofs-core", version = "~14.0.0", default-features = false}
 sha2raw = { path = "../sha2raw", version = "~9.0.0"}
 filecoin-hashers = { path = "../filecoin-hashers", version = "~9.0.0", default-features = false, features = ["poseidon", "sha256"]}
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 memmap2 = "0.5.6"
 num-bigint = "0.4.3"
 num-traits = "0.2"

storage-proofs-porep/src/stacked/vanilla/proof.rs

@@ -452,7 +452,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr
     }
 
     #[cfg(any(feature = "cuda", feature = "opencl"))]
-    fn generate_tree_c<ColumnArity, TreeArity>(
+    pub fn generate_tree_c<ColumnArity, TreeArity>(
         nodes_count: usize,
         tree_count: usize,
         configs: Vec<StoreConfig>,
@@ -480,7 +480,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr
     }
 
     #[cfg(not(any(feature = "cuda", feature = "opencl")))]
-    fn generate_tree_c<ColumnArity, TreeArity>(
+    pub fn generate_tree_c<ColumnArity, TreeArity>(
         nodes_count: usize,
         tree_count: usize,
         configs: Vec<StoreConfig>,

storage-proofs-update/Cargo.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 storage-proofs-core = { path = "../storage-proofs-core", version = "~14.0.0", default-features = false}
 storage-proofs-porep = { path = "../storage-proofs-porep", version = "~14.0.0", default-features = false}
 filecoin-hashers = { path = "../filecoin-hashers", version = "~9.0.0", default-features = false, features = ["poseidon", "sha256"]}
-merkletree = "0.22.0"
+merkletree = "0.23.0"
 rayon = "1.0.0"
 serde = { version = "1.0", features = ["derive"]}
 ff = "0.13.0"