diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 981370de8..196bba978 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -82,3 +82,7 @@ blst = [ [[bench]] name = "preprocessing" harness = false + +[[bench]] +name = "aggregation" +harness = false diff --git a/filecoin-proofs/benches/aggregation.rs b/filecoin-proofs/benches/aggregation.rs new file mode 100644 index 000000000..88ca02974 --- /dev/null +++ b/filecoin-proofs/benches/aggregation.rs @@ -0,0 +1,166 @@ +use std::sync::Once; +use std::time::Duration; + +use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; +use filecoin_proofs::{ + caches::{get_stacked_srs_key, get_stacked_srs_verifier_key}, + get_seal_inputs, PoRepConfig, PoRepProofPartitions, SectorShape2KiB, SectorShape32GiB, + SectorSize, POREP_PARTITIONS, SECTOR_SIZE_2_KIB, SECTOR_SIZE_32_GIB, +}; +use rand::{thread_rng, Rng}; +use storage_proofs_core::{api_version::ApiVersion, is_legacy_porep_id}; + +static INIT_LOGGER: Once = Once::new(); +fn init_logger() { + INIT_LOGGER.call_once(|| { + fil_logger::init(); + }); +} + +fn bench_seal_inputs(c: &mut Criterion) { + let params = vec![1, 256, 512, 1024]; + + let mut rng = thread_rng(); + + let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value + + let mut porep_id = [0u8; 32]; + porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); + assert!(!is_legacy_porep_id(porep_id)); + + let config = PoRepConfig { + sector_size: SectorSize(SECTOR_SIZE_2_KIB), + partitions: PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(&SECTOR_SIZE_2_KIB) + .expect("unknown sector size"), + ), + porep_id, + api_version: ApiVersion::V1_1_0, + }; + let comm_r = [5u8; 32]; + let comm_d = [6u8; 32]; + let prover_id = [7u8; 32]; + + let ticket = rng.gen(); + let seed = rng.gen(); + let sector_id = rng.gen::().into(); + + let mut group = c.benchmark_group("bench_seal_inputs"); + for iterations in params { + group + .bench_function(format!("get-seal-inputs-{}", iterations), |b| { + b.iter(|| { + for _ in 0..iterations { + get_seal_inputs::( + config, comm_r, comm_d, prover_id, sector_id, ticket, seed, + ) + .expect("get seal inputs failed"); + } + }); + }) + .sample_size(10) + .throughput(Throughput::Bytes(iterations as u64)) + .warm_up_time(Duration::from_secs(1)); + } + + group.finish(); +} + +fn bench_stacked_srs_key(c: &mut Criterion) { + init_logger(); + let params = vec![128, 256, 512, 1024]; + + let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value + + let mut porep_id = [0u8; 32]; + porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); + assert!(!is_legacy_porep_id(porep_id)); + + let config = PoRepConfig { + sector_size: SectorSize(SECTOR_SIZE_32_GIB), + partitions: PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(&SECTOR_SIZE_32_GIB) + .expect("unknown sector size"), + ), + porep_id, + api_version: ApiVersion::V1_1_0, + }; + + let mut group = c.benchmark_group("bench-stacked-srs-key"); + for num_proofs_to_aggregate in params { + group.bench_function( + format!("get-stacked-srs-key-{}", num_proofs_to_aggregate), + |b| { + b.iter(|| { + black_box( + get_stacked_srs_key::(config, num_proofs_to_aggregate) + .expect("get stacked srs key failed"), + ) + }) + }, + ); + } + + group.finish(); +} + +fn bench_stacked_srs_verifier_key(c: &mut Criterion) { + init_logger(); + let params = vec![128, 256, 512, 1024]; + + let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value + + let mut porep_id = [0u8; 32]; + porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); + assert!(!is_legacy_porep_id(porep_id)); + + let config = PoRepConfig { + sector_size: SectorSize(SECTOR_SIZE_32_GIB), + partitions: PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(&SECTOR_SIZE_32_GIB) + .expect("unknown sector size"), + ), + porep_id, + api_version: ApiVersion::V1_1_0, + }; + + let mut group = c.benchmark_group("bench-stacked-srs-verifier-key"); + for num_proofs_to_aggregate in params { + group + .bench_function( + format!("get-stacked-srs-verifier-key-{}", num_proofs_to_aggregate), + |b| { + b.iter(|| { + black_box( + get_stacked_srs_verifier_key::( + config, + num_proofs_to_aggregate, + ) + .expect("get stacked srs key failed"), + ) + }) + }, + ) + .sample_size(10) + .warm_up_time(Duration::from_secs(1)); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_seal_inputs, + bench_stacked_srs_key, + bench_stacked_srs_verifier_key, +); +criterion_main!(benches); diff --git a/filecoin-proofs/src/lib.rs b/filecoin-proofs/src/lib.rs index d6a7c9580..5488fab86 100644 --- a/filecoin-proofs/src/lib.rs +++ b/filecoin-proofs/src/lib.rs @@ -3,6 +3,7 @@ #![warn(clippy::unnecessary_wraps)] #![allow(clippy::upper_case_acronyms)] +pub mod caches; pub mod constants; pub mod param; pub mod parameters; @@ -10,7 +11,6 @@ pub mod pieces; pub mod types; mod api; -mod caches; mod commitment_reader; pub use api::*; diff --git a/filecoin-proofs/tests/aggregate_proof_bytes b/filecoin-proofs/tests/aggregate_proof_bytes new file mode 100644 index 000000000..86dc9efff Binary files /dev/null and b/filecoin-proofs/tests/aggregate_proof_bytes differ diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index fa1517776..b972626e8 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -5,7 +5,9 @@ use std::path::{Path, PathBuf}; use std::sync::Once; use anyhow::{ensure, Result}; -use bellperson::bls::Fr; +use bellperson::bls::{Bls12, Fr}; +use bellperson::groth16; +use bincode::serialize; use ff::Field; use filecoin_hashers::Hasher; use filecoin_proofs::{ @@ -221,14 +223,54 @@ fn test_seal_proof_aggregation_1_2kib_porep_id_v1_1_base_8() -> Result<()> { #[ignore] fn test_seal_proof_aggregation_3_2kib_porep_id_v1_1_base_8() -> Result<()> { let proofs_to_aggregate = 3; // Requires auto-padding - inner_test_seal_proof_aggregation_2kib_porep_id_v1_1_base_8(proofs_to_aggregate) + + let porep_id = ARBITRARY_POREP_ID_V1_1_0; + assert!(!is_legacy_porep_id(porep_id)); + let verified = aggregate_proofs::( + SECTOR_SIZE_2_KIB, + &porep_id, + ApiVersion::V1_1_0, + proofs_to_aggregate, + )?; + assert!(verified); + + Ok(()) } #[test] #[ignore] fn test_seal_proof_aggregation_5_2kib_porep_id_v1_1_base_8() -> Result<()> { let proofs_to_aggregate = 5; // Requires auto-padding - inner_test_seal_proof_aggregation_2kib_porep_id_v1_1_base_8(proofs_to_aggregate) + + let porep_id = ARBITRARY_POREP_ID_V1_1_0; + assert!(!is_legacy_porep_id(porep_id)); + let verified = aggregate_proofs::( + SECTOR_SIZE_2_KIB, + &porep_id, + ApiVersion::V1_1_0, + proofs_to_aggregate, + )?; + assert!(verified); + + Ok(()) +} + +#[test] +#[ignore] +fn test_seal_proof_aggregation_257_2kib_porep_id_v1_1_base_8() -> Result<()> { + let proofs_to_aggregate = 257; // Requires auto-padding + + let porep_id = ARBITRARY_POREP_ID_V1_1_0; + assert!(!is_legacy_porep_id(porep_id)); + let verified = aggregate_proofs::( + SECTOR_SIZE_2_KIB, + &porep_id, + ApiVersion::V1_1_0, + proofs_to_aggregate, + )?; + assert!(verified); + + Ok(()) } #[test] @@ -267,6 +309,60 @@ fn test_seal_proof_aggregation_1_32kib_porep_id_v1_1_base_8() -> Result<()> { Ok(()) } +#[test] +#[ignore] +fn test_seal_proof_aggregation_818_32kib_porep_id_v1_1_base_8() -> Result<()> { + let proofs_to_aggregate = 818; // Requires auto-padding + + let porep_id = ARBITRARY_POREP_ID_V1_1_0; + assert!(!is_legacy_porep_id(porep_id)); + let verified = aggregate_proofs::( + SECTOR_SIZE_32_KIB, + &porep_id, + ApiVersion::V1_1_0, + proofs_to_aggregate, + )?; + assert!(verified); + + Ok(()) +} + +//#[test] +//#[ignore] +//fn test_seal_proof_aggregation_818_32gib_porep_id_v1_1_base_8() -> Result<()> { +// let proofs_to_aggregate = 818; // Requires auto-padding +// +// let porep_id = ARBITRARY_POREP_ID_V1_1_0; +// assert!(!is_legacy_porep_id(porep_id)); +// let verified = aggregate_proofs::( +// SECTOR_SIZE_32_GIB, +// &porep_id, +// ApiVersion::V1_1_0, +// proofs_to_aggregate, +// )?; +// assert!(verified); +// +// Ok(()) +//} + +//#[test] +//#[ignore] +//fn test_seal_proof_aggregation_818_64gib_porep_id_v1_1_base_8() -> Result<()> { +// let proofs_to_aggregate = 818; // Requires auto-padding +// +// let porep_id = ARBITRARY_POREP_ID_V1_1_0; +// assert!(!is_legacy_porep_id(porep_id)); +// let verified = aggregate_proofs::( +// SECTOR_SIZE_64_GIB, +// &porep_id, +// ApiVersion::V1_1_0, +// proofs_to_aggregate, +// )?; +// assert!(verified); +// +// Ok(()) +//} + //#[test] //#[ignore] //fn test_seal_proof_aggregation_1024_2kib_porep_id_v1_1_base_8() -> Result<()> { @@ -281,61 +377,6 @@ fn test_seal_proof_aggregation_1_32kib_porep_id_v1_1_base_8() -> Result<()> { // inner_test_seal_proof_aggregation_2kib_porep_id_v1_1_base_8(proofs_to_aggregate) //} -fn inner_test_seal_proof_aggregation_2kib_porep_id_v1_1_base_8( - proofs_to_aggregate: usize, -) -> Result<()> { - let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value - - let mut porep_id = [0u8; 32]; - porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); - assert!(!is_legacy_porep_id(porep_id)); - - let rng = &mut XorShiftRng::from_seed(TEST_SEED); - let prover_fr: DefaultTreeDomain = Fr::random(rng).into(); - let mut prover_id = [0u8; 32]; - prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr)); - - let mut commit_outputs = Vec::with_capacity(proofs_to_aggregate); - let mut commit_inputs = Vec::with_capacity(proofs_to_aggregate); - let mut seeds = Vec::with_capacity(proofs_to_aggregate); - let mut comm_rs = Vec::with_capacity(proofs_to_aggregate); - - let (commit_output, commit_input, seed, comm_r) = - create_seal_for_aggregation::<_, SectorShape2KiB>( - rng, - SECTOR_SIZE_2_KIB, - prover_id, - &porep_id, - ApiVersion::V1_1_0, - )?; - - // duplicate a single proof to desired target for aggregation - for _ in 0..proofs_to_aggregate { - commit_outputs.push(commit_output.clone()); - commit_inputs.extend(commit_input.clone()); - seeds.push(seed); - comm_rs.push(comm_r); - } - - let config = porep_config(SECTOR_SIZE_2_KIB, porep_id, ApiVersion::V1_1_0); - let aggregate_proof = aggregate_seal_commit_proofs::( - config, - &comm_rs, - &seeds, - commit_outputs.as_slice(), - )?; - let verified = verify_aggregate_seal_commit_proofs::( - config, - aggregate_proof, - &comm_rs, - &seeds, - commit_inputs, - )?; - assert!(verified); - - Ok(()) -} - fn aggregate_proofs( sector_size: u64, porep_id: &[u8; 32], @@ -352,16 +393,12 @@ fn aggregate_proofs( let mut seeds = Vec::with_capacity(num_proofs_to_aggregate); let mut comm_rs = Vec::with_capacity(num_proofs_to_aggregate); + let (commit_output, commit_input, seed, comm_r) = + create_seal_for_aggregation::<_, Tree>(rng, sector_size, prover_id, porep_id, api_version)?; + for _ in 0..num_proofs_to_aggregate { - let (commit_output, commit_input, seed, comm_r) = create_seal_for_aggregation::<_, Tree>( - rng, - sector_size, - prover_id, - porep_id, - api_version, - )?; - commit_outputs.push(commit_output); - commit_inputs.extend(commit_input); + commit_outputs.push(commit_output.clone()); + commit_inputs.extend(commit_input.clone()); seeds.push(seed); comm_rs.push(comm_r); } @@ -1474,3 +1511,39 @@ fn create_fake_seal( Ok((sector_id, sealed_sector_file, comm_r, cache_dir)) } + +#[test] +fn test_aggregate_proof_encode_decode() -> Result<()> { + // This byte vector is a natively serialized aggregate proof generated from the + // 'test_seal_proof_aggregation_257_2kib_porep_id_v1_1_base_8' test. + let aggregate_proof_bytes = std::include_bytes!("./aggregate_proof_bytes"); + let expected_aggregate_proof_len = 29_044; + + // Re-construct the aggregate proof from the bytes, using the native deserialization method. + let aggregate_proof: groth16::aggregate::AggregateProof = + groth16::aggregate::AggregateProof::read(std::io::Cursor::new(&aggregate_proof_bytes))?; + let aggregate_proof_count = aggregate_proof.tmipp.gipa.nproofs as usize; + let expected_aggregate_proof_count = 512; + + assert_eq!(aggregate_proof_count, expected_aggregate_proof_count); + + // Re-serialize the proof to ensure a round-trip match. + let mut aggregate_proof_bytes2 = Vec::new(); + aggregate_proof.write(&mut aggregate_proof_bytes2)?; + + assert_eq!(aggregate_proof_bytes.len(), expected_aggregate_proof_len); + assert_eq!(aggregate_proof_bytes.len(), aggregate_proof_bytes2.len()); + assert_eq!(aggregate_proof_bytes, aggregate_proof_bytes2.as_slice()); + + // Note: the native serialization format is more compact than bincode serialization, so assert that here. + let bincode_serialized_proof = serialize(&aggregate_proof)?; + let expected_bincode_serialized_proof_len = 56_436; + + assert!(aggregate_proof_bytes2.len() < bincode_serialized_proof.len()); + assert_eq!( + bincode_serialized_proof.len(), + expected_bincode_serialized_proof_len + ); + + Ok(()) +} diff --git a/storage-proofs-core/src/compound_proof.rs b/storage-proofs-core/src/compound_proof.rs index 6c46823ce..6532a835f 100644 --- a/storage-proofs-core/src/compound_proof.rs +++ b/storage-proofs-core/src/compound_proof.rs @@ -278,14 +278,18 @@ where } /// Given a prover_srs key, a list of groth16 proofs, and an ordered list of seeds - /// (used to derive the PoRep challenges) hashed using FIXME, aggregate them all into + /// (used to derive the PoRep challenges) hashed pair-wise with the comm_rs using sha256, aggregate them all into /// an AggregateProof type. fn aggregate_proofs( prover_srs: &ProverSRS, - hashed_seeds: &[u8], + hashed_seeds_and_comm_rs: &[u8], proofs: &[groth16::Proof], ) -> Result> { - Ok(aggregate_proofs::(prover_srs, hashed_seeds, proofs)?) + Ok(aggregate_proofs::( + prover_srs, + hashed_seeds_and_comm_rs, + proofs, + )?) } /// Verifies the aggregate proof, with respect to the flattened input list. @@ -298,7 +302,7 @@ where fn verify_aggregate_proofs( ip_verifier_srs: &VerifierSRS, pvk: &PreparedVerifyingKey, - hashed_seeds: &[u8], + hashed_seeds_and_comm_rs: &[u8], public_inputs: &[Vec], aggregate_proof: &groth16::aggregate::AggregateProof, ) -> Result { @@ -310,7 +314,7 @@ where &mut rng, public_inputs, aggregate_proof, - hashed_seeds, + hashed_seeds_and_comm_rs, )?) }