diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a602c4efc..64c99feeb2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: no_output_timeout: 30m - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - run: rustup install $(cat rust-toolchain) - run: rustup default $(cat rust-toolchain) - run: rustup component add rustfmt-preview @@ -40,7 +40,7 @@ jobs: paths: - Cargo.lock - save_cache: - key: cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + key: cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} paths: - /root/.cargo - /root/.rustup @@ -56,7 +56,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: name: Test (stable) @@ -85,7 +85,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: name: Test (stable) in release profile @@ -111,7 +111,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: name: Test ignored in release profile @@ -134,7 +134,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: name: Test (nightly) @@ -153,7 +153,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: name: Benchmarks (nightly) @@ -241,7 +241,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - run: name: Run cargo fmt command: cargo fmt --all -- --check @@ -258,7 +258,7 @@ jobs: at: "." - restore_cache: keys: - - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} + - cargo-v27b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - run: name: Run cargo clippy command: cargo +$(cat rust-toolchain) clippy --all @@ -319,7 +319,7 @@ commands: save_parameter_cache: steps: - save_cache: - key: proof-params-v26d-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }} + key: proof-params-v27b-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }} paths: - "~/paramcache.awesome" - "~/filecoin-proof-parameters/" @@ -328,7 +328,7 @@ commands: - configure_environment_variables - restore_cache: keys: - - proof-params-v26d-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }} + - proof-params-v27b-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }} configure_environment_variables: steps: - run: diff --git a/storage-proofs/core/src/gadgets/insertion.rs b/storage-proofs/core/src/gadgets/insertion.rs index 9bdb0f54f7..5dca199554 100644 --- a/storage-proofs/core/src/gadgets/insertion.rs +++ b/storage-proofs/core/src/gadgets/insertion.rs @@ -3,7 +3,7 @@ //! Insert an `AllocatedNum` into a sequence of `AllocatedNums` at an arbitrary position. //! This can be thought of as a generalization of `AllocatedNum::conditionally_reverse` and reduces to it in the binary case. -use bellperson::gadgets::boolean::Boolean; +use bellperson::gadgets::boolean::{AllocatedBit, Boolean}; use bellperson::gadgets::num::AllocatedNum; use bellperson::{ConstraintSystem, SynthesisError}; use ff::Field; @@ -21,6 +21,22 @@ pub fn insert>( let size = elements.len() + 1; assert_eq!(1 << bits.len(), size); + // For the sizes we know we need, we can take advantage of redundancy in the candidate selection at each position. + // This allows us to accomplish insertion with fewer constraints, if we hand-optimize. + // We don't need a special case for size 2 because the general algorithm + // collapses to `conditionally_reverse` when size = 2. + // + // If no special cases have been hand-coded, use the general algorithm. + // This costs size * (size - 1) constraints. + // + // Future work: In theory, we could compile arbitrary lookup tables to minimize constraints and avoid + // the most general case except when actually required — which it never is for simple insertion. + if size == 4 { + return insert_4(cs, element, bits, elements); + } else if size == 8 { + return insert_8(cs, element, bits, elements); + }; + // Running example choices, represent inserting x into [1, 2, 3]. // An indexed sequence of correct results, one of which (the indexed one) will be selected. @@ -69,6 +85,170 @@ pub fn insert>( Ok(result) } +pub fn insert_4>( + cs: &mut CS, + element: &AllocatedNum, + bits: &[Boolean], + elements: &[AllocatedNum], +) -> Result>, SynthesisError> { + assert_eq!(elements.len() + 1, 4); + assert_eq!(bits.len(), 2); + + /* + To insert A into [b, c, d] at position n of bits, represented by booleans [b0, b1, b2]. + n [b0, b1] pos 0 1 2 3 + 0 [0, 0] A b c d + 1 [1, 0] b A c d + 2 [0, 1] b c A d + 3 [1, 1] b c d A + + A = element + b = elements[0] + c = elements[1] + d = elements[2] + */ + let (b0, b1) = (&bits[0], &bits[1]); + let (a, b, c, d) = (&element, &elements[0], &elements[1], &elements[2]); + + /// Define witness macro to allow legible definition of positional constraints. + /// See example expansions in comment to first usages below. + macro_rules! witness { + ( $var:ident <== if $cond:ident { $a:expr } else { $b:expr }) => { + let $var = pick(cs.namespace(|| stringify!($var)), $cond, $a, $b)?; + }; + } + + // Witness naming convention: + // `p0_x0` means "Output position 0 when b0 is unknown (x) and b1 is 0." + + // Declaration: + witness!(p0_x0 <== if b0 { b } else { a }); + witness!(p0 <== if b1 { b } else { &p0_x0 }); + // Expansion: + // let p0_x0 = pick(cs.namespace(|| "p0_x0"), b0, b, a)?; + // let p0 = pick(cs.namespace(|| "p0"), b1, b, &p0_x0)?; + + witness!(p1_x0 <== if b0 { a } else { b }); + witness!(p1 <== if b1 { c } else { &p1_x0 }); + + witness!(p2_x1 <== if b0 { d } else { a }); + witness!(p2 <== if b1 { &p2_x1 } else {c }); + + witness!(p3_x1 <== if b0 { a } else { d }); + witness!(p3 <== if b1 { &p3_x1 } else { d }); + + Ok(vec![p0, p1, p2, p3]) +} + +pub fn insert_8>( + cs: &mut CS, + element: &AllocatedNum, + bits: &[Boolean], + elements: &[AllocatedNum], +) -> Result>, SynthesisError> { + assert_eq!(elements.len() + 1, 8); + assert_eq!(bits.len(), 3); + /* + To insert A into [b, c, d, e, f, g, h] at position n of bits, represented by booleans [b0, b1, b2]. + n [b0, b1, b2] pos 0 1 2 3 4 5 6 7 + 0 [0, 0, 0] A b c d e f g h + 1 [1, 0, 0] b A c d e f g h + 2 [0, 1, 0] b c A d e f g h + 3 [1, 1, 0] b c d A e f g h + 4 [0, 0, 1] b c d e A f g h + 5 [1, 0, 1] b c d e f A g h + 6 [0, 1, 1] b c d e f g A h + 7 [1, 1, 1] b c d e f g h A + + + A = element + b = elements[0] + c = elements[1] + d = elements[2] + e = elements[3] + f = elements[4] + g = elements[5] + h = elements[6] + */ + + let (b0, b1, b2) = (&bits[0], &bits[1], &bits[2]); + let (a, b, c, d, e, f, g, h) = ( + &element, + &elements[0], + &elements[1], + &elements[2], + &elements[3], + &elements[4], + &elements[5], + &elements[6], + ); + + // true if booleans b0 and b1 are both false: `(not b0) and (not b1)` + // (1 - b0) * (1 - b1) = 1 + let b0_nor_b1 = match (b0, b1) { + (Boolean::Is(ref b0), Boolean::Is(ref b1)) => { + Boolean::Is(AllocatedBit::nor(cs.namespace(|| "b0 nor b1"), b0, b1)?) + } + _ => panic!("bits must be allocated and unnegated"), + }; + + // true if booleans b0 and b1 are both true: `b0 and b1` + // b0 * b1 = 1 + let b0_and_b1 = match (&bits[0], &bits[1]) { + (Boolean::Is(ref b0), Boolean::Is(ref b1)) => { + Boolean::Is(AllocatedBit::and(cs.namespace(|| "b0 and b1"), b0, b1)?) + } + _ => panic!("bits must be allocated and unnegated"), + }; + + /// Define witness macro to allow legible definition of positional constraints. + /// See example expansions in comment to first usages below. + macro_rules! witness { + ( $var:ident <== if $cond:ident { $a:expr } else { $b:expr }) => { + let $var = pick(cs.namespace(|| stringify!($var)), $cond, $a, $b)?; + }; + + // Match condition terms which are explict syntactic references. + ( $var:ident <== if &$cond:ident { $a:expr } else { $b:expr }) => { + let $var = pick(cs.namespace(|| stringify!($var)), &$cond, $a, $b)?; + }; + } + + // Declaration: + witness!(p0_xx0 <== if &b0_nor_b1 { a } else { b }); + witness!(p0 <== if b2 { b } else { &p0_xx0 }); + // Expansion: + // let p0_xx0 = pick(cs.namespace(|| "p0_xx0"), &b0_nor_b1, a, b)?; + // let p0 = pick(cs.namespace(|| "p0"), b2, b, &p0_xx0)?; + + witness!(p1_x00 <== if b0 { a } else { b }); + witness!(p1_xx0 <== if b1 { c } else { &p1_x00 }); + witness!(p1 <== if b2 { c } else { &p1_xx0 }); + + witness!(p2_x10 <== if b0 { d } else { a }); + witness!(p2_xx0 <== if b1 { &p2_x10 } else { c }); + witness!(p2 <== if b2 { d } else { &p2_xx0 }); + + witness!(p3_xx0 <== if &b0_and_b1 { a } else { d }); + witness!(p3 <== if b2 { e } else { &p3_xx0 }); + + witness!(p4_xx1 <== if &b0_nor_b1 { a } else { f }); + witness!(p4 <== if b2 { &p4_xx1 } else { e }); + + witness!(p5_x01 <== if b0 { a } else { f }); + witness!(p5_xx1 <== if b1 { g } else { &p5_x01 }); + witness!(p5 <== if b2 { &p5_xx1 } else { f }); + + witness!(p6_x11 <== if b0 { h } else { a }); + witness!(p6_xx1 <== if b1 { &p6_x11 } else { g }); + witness!(p6 <== if b2 { &p6_xx1 } else { g }); + + witness!(p7_xx1 <== if &b0_and_b1 { a } else { h }); + witness!(p7 <== if b2 { &p7_xx1 } else { h }); + + Ok(vec![p0, p1, p2, p3, p4, p5, p6, p7]) +} + /// Select the nth element of `from`, where `path_bits` represents n, least-significant bit first. /// The returned result contains the selected element, and constraints are enforced. /// `from.len()` must be a power of two. @@ -125,6 +305,8 @@ where } })?; + // Constrain (b - a) * condition = (b - c), ensuring c = a iff + // condition is true, otherwise c = b. cs.enforce( || "pick", |lc| lc + b.get_variable() - a.get_variable(), @@ -221,7 +403,8 @@ mod tests { let to_insert = AllocatedNum::::alloc(&mut cs.namespace(|| "insert"), || { - Ok(::random(rng)) + let elt_to_insert = ::random(rng); + Ok(elt_to_insert) }) .unwrap(); @@ -256,12 +439,15 @@ mod tests { for i in 0..size - 1 { let a = elements[i].get_value(); let b = inserted[i].get_value(); - assert_eq!(a, b) } // One selection for each element of the result. - let expected_constraints = size * (size - 1); + let expected_constraints = match size { + 8 => 22, // unoptimized, would be 56 + 4 => 8, // unoptimized, would be 12 + _ => size * (size - 1), + }; let actual_constraints = cs.num_constraints() - test_constraints; assert_eq!(expected_constraints, actual_constraints); diff --git a/storage-proofs/core/src/gadgets/por.rs b/storage-proofs/core/src/gadgets/por.rs index 40ba0c43c6..94fe5c8096 100644 --- a/storage-proofs/core/src/gadgets/por.rs +++ b/storage-proofs/core/src/gadgets/por.rs @@ -546,18 +546,18 @@ mod tests { #[test] fn test_por_circuit_pedersen_base_4() { - test_por_circuit::>(3, 12_411); + test_por_circuit::>(3, 12_399); } #[test] fn test_por_circuit_pedersen_sub_8_2() { - test_por_circuit::>(3, 20_731); + test_por_circuit::>(3, 20_663); } #[test] fn test_por_circuit_pedersen_top_8_4_2() { test_por_circuit::>( - 3, 24_867, + 3, 24_795, ); } @@ -566,67 +566,67 @@ mod tests { // We can handle top-heavy trees with a non-zero subtree arity. // These should never be produced, though. test_por_circuit::>( - 3, 24_867, + 3, 24_795, ); } #[test] fn test_por_circuit_blake2s_base_4() { - test_por_circuit::>(3, 130_308); + test_por_circuit::>(3, 130_296); } #[test] fn test_por_circuit_sha256_base_4() { - test_por_circuit::>(3, 216_270); + test_por_circuit::>(3, 216_258); } #[test] fn test_por_circuit_poseidon_base_4() { - test_por_circuit::>(3, 1_185); + test_por_circuit::>(3, 1_173); } #[test] fn test_por_circuit_pedersen_base_8() { - test_por_circuit::>(3, 19_357); + test_por_circuit::>(3, 19_289); } #[test] fn test_por_circuit_blake2s_base_8() { - test_por_circuit::>(3, 174_571); + test_por_circuit::>(3, 174_503); } #[test] fn test_por_circuit_sha256_base_8() { - test_por_circuit::>(3, 251_055); + test_por_circuit::>(3, 250_987); } #[test] fn test_por_circuit_poseidon_base_8() { - test_por_circuit::>(3, 1_137); + test_por_circuit::>(3, 1_069); } #[test] fn test_por_circuit_poseidon_sub_8_2() { - test_por_circuit::>(3, 1_454); + test_por_circuit::>(3, 1_386); } #[test] fn test_por_circuit_poseidon_top_8_4_2() { test_por_circuit::>( - 3, 1_848, + 3, 1_776, ); } #[test] fn test_por_circuit_poseidon_top_8_8() { // This is the shape we want for 32GiB sectors. - test_por_circuit::>(3, 1_704); + test_por_circuit::>(3, 1_602); } #[test] fn test_por_circuit_poseidon_top_8_8_2() { // This is the shape we want for 64GiB secotrs. test_por_circuit::>( - 3, 2_021, + 3, 1_919, ); } @@ -635,7 +635,7 @@ mod tests { // We can handle top-heavy trees with a non-zero subtree arity. // These should never be produced, though. test_por_circuit::>( - 3, 1_848, + 3, 1_776, ); } @@ -893,12 +893,12 @@ mod tests { #[test] fn test_private_por_input_circuit_pedersen_quad() { - test_private_por_input_circuit::>(12_410); + test_private_por_input_circuit::>(12_398); } #[test] fn test_private_por_input_circuit_poseidon_quad() { - test_private_por_input_circuit::>(1_184); + test_private_por_input_circuit::>(1_172); } fn test_private_por_input_circuit(num_constraints: usize) { diff --git a/storage-proofs/core/src/lib.rs b/storage-proofs/core/src/lib.rs index cdd9759ac1..83d30263f2 100644 --- a/storage-proofs/core/src/lib.rs +++ b/storage-proofs/core/src/lib.rs @@ -1,4 +1,5 @@ #![deny(clippy::all, clippy::perf, clippy::correctness)] +#![allow(clippy::many_single_char_names)] #![allow(clippy::unreadable_literal)] #![allow(clippy::type_repetition_in_bounds)] diff --git a/storage-proofs/core/src/parameter_cache.rs b/storage-proofs/core/src/parameter_cache.rs index 9fa480b552..5c8fa8d398 100644 --- a/storage-proofs/core/src/parameter_cache.rs +++ b/storage-proofs/core/src/parameter_cache.rs @@ -16,7 +16,7 @@ use std::io::{self, SeekFrom}; use std::path::{Path, PathBuf}; /// Bump this when circuits change to invalidate the cache. -pub const VERSION: usize = 26; +pub const VERSION: usize = 27; pub const PARAMETER_CACHE_ENV_VAR: &str = "FIL_PROOFS_PARAMETER_CACHE"; pub const PARAMETER_CACHE_DIR: &str = "/var/tmp/filecoin-proof-parameters/"; diff --git a/storage-proofs/porep/src/stacked/circuit/proof.rs b/storage-proofs/porep/src/stacked/circuit/proof.rs index e6db9700ca..592dde6363 100644 --- a/storage-proofs/porep/src/stacked/circuit/proof.rs +++ b/storage-proofs/porep/src/stacked/circuit/proof.rs @@ -376,17 +376,17 @@ mod tests { #[test] fn stacked_input_circuit_poseidon_base_8() { - stacked_input_circuit::>(22, 1_200_258); + stacked_input_circuit::>(22, 1_199_714); } #[test] fn stacked_input_circuit_poseidon_sub_8_4() { - stacked_input_circuit::>(22, 1_297_326); + stacked_input_circuit::>(22, 1_296_718); } #[test] fn stacked_input_circuit_poseidon_top_8_4_2() { - stacked_input_circuit::>(22, 1_347_780); + stacked_input_circuit::>(22, 1_347_172); } fn stacked_input_circuit( diff --git a/storage-proofs/post/src/election/circuit.rs b/storage-proofs/post/src/election/circuit.rs index 83f0d55f07..928a3a683b 100644 --- a/storage-proofs/post/src/election/circuit.rs +++ b/storage-proofs/post/src/election/circuit.rs @@ -197,12 +197,12 @@ mod tests { #[test] fn test_election_post_circuit_pedersen() { - test_election_post_circuit::>(389_883); + test_election_post_circuit::>(388_523); } #[test] fn test_election_post_circuit_poseidon() { - test_election_post_circuit::>(24_426); + test_election_post_circuit::>(23_066); } fn test_election_post_circuit(expected_constraints: usize) { diff --git a/storage-proofs/post/src/fallback/circuit.rs b/storage-proofs/post/src/fallback/circuit.rs index 31b450995f..7dc5bea651 100644 --- a/storage-proofs/post/src/fallback/circuit.rs +++ b/storage-proofs/post/src/fallback/circuit.rs @@ -191,37 +191,37 @@ mod tests { #[test] fn fallback_post_pedersen_single_partition_matching_base_8() { - fallback_post::>(3, 3, 1, 19, 294_459); + fallback_post::>(3, 3, 1, 19, 293_439); } #[test] fn fallback_post_poseidon_single_partition_matching_base_8() { - fallback_post::>(3, 3, 1, 19, 17_988); + fallback_post::>(3, 3, 1, 19, 16_968); } #[test] fn fallback_post_poseidon_single_partition_matching_sub_8_4() { - fallback_post::>(3, 3, 1, 19, 23_898); + fallback_post::>(3, 3, 1, 19, 22_818); } #[test] fn fallback_post_poseidon_single_partition_matching_top_8_4_2() { - fallback_post::>(3, 3, 1, 19, 28_653); + fallback_post::>(3, 3, 1, 19, 27_573); } #[test] fn fallback_post_poseidon_single_partition_smaller_base_8() { - fallback_post::>(2, 3, 1, 19, 17_988); + fallback_post::>(2, 3, 1, 19, 16_968); } #[test] fn fallback_post_poseidon_two_partitions_matching_base_8() { - fallback_post::>(4, 2, 2, 13, 11_992); + fallback_post::>(4, 2, 2, 13, 11_312); } #[test] fn fallback_post_poseidon_two_partitions_smaller_base_8() { - fallback_post::>(5, 3, 2, 19, 17_988); + fallback_post::>(5, 3, 2, 19, 16_968); } #[test] @@ -242,7 +242,7 @@ mod tests { .synthesize(&mut cs) .unwrap(); - assert_eq!(cs.num_constraints(), 285_180); + assert_eq!(cs.num_constraints(), 268_180); } fn fallback_post(