diff --git a/.circleci/config.yml b/.circleci/config.yml index 69e1cdd597..a472a2895a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -622,3 +622,15 @@ workflows: branches: only: - master + + - test: + name: test_fr32 + crate: "fr32" + requires: + - cargo_fetch + + - test_blst: + name: test_blst_fr32 + crate: "fr32" + requires: + - cargo_fetch diff --git a/Cargo.toml b/Cargo.toml index 05ea8b1f39..bd455d7c81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "storage-proofs/post", "fil-proofs-tooling", "fil-proofs-param", + "fr32", "sha2raw", "filecoin-hashers", ] diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index e131cd8f9d..774a33972e 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -37,6 +37,7 @@ gperftools = { version = "0.2", optional = true } generic-array = "0.14.4" groupy = "0.3.0" byte-slice-cast = "1.0.0" +fr32 = { path = "../fr32", default-features = false } [dev-dependencies] criterion = "0.3" @@ -51,9 +52,9 @@ cpu-profile = ["gperftools"] heap-profile = ["gperftools/heap"] simd = ["storage-proofs/simd"] asm = ["storage-proofs/asm"] -gpu = ["storage-proofs/gpu", "bellperson/gpu", "filecoin-hashers/gpu"] -pairing = ["storage-proofs/pairing", "bellperson/pairing", "filecoin-hashers/pairing"] -blst = ["storage-proofs/blst", "bellperson/blst", "filecoin-hashers/blst"] +gpu = ["storage-proofs/gpu", "bellperson/gpu", "filecoin-hashers/gpu", "fr32/gpu"] +pairing = ["storage-proofs/pairing", "bellperson/pairing", "filecoin-hashers/pairing", "fr32/pairing"] +blst = ["storage-proofs/blst", "bellperson/blst", "filecoin-hashers/blst", "fr32/blst"] [[bench]] name = "preprocessing" diff --git a/filecoin-proofs/benches/preprocessing.rs b/filecoin-proofs/benches/preprocessing.rs index ad66aaf2db..8f321ae3dc 100644 --- a/filecoin-proofs/benches/preprocessing.rs +++ b/filecoin-proofs/benches/preprocessing.rs @@ -2,7 +2,8 @@ use std::io::{self, Read}; use std::time::Duration; use criterion::{criterion_group, criterion_main, Criterion, ParameterizedBenchmark, Throughput}; -use filecoin_proofs::{add_piece, fr32_reader::Fr32Reader, PaddedBytesAmount, UnpaddedBytesAmount}; +use filecoin_proofs::{add_piece, PaddedBytesAmount, UnpaddedBytesAmount}; +use fr32::Fr32Reader; use rand::{thread_rng, Rng}; #[cfg(feature = "cpu-profile")] diff --git a/filecoin-proofs/src/api/mod.rs b/filecoin-proofs/src/api/mod.rs index 22ccb1d6e2..b746d983da 100644 --- a/filecoin-proofs/src/api/mod.rs +++ b/filecoin-proofs/src/api/mod.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use anyhow::{ensure, Context, Result}; use bincode::deserialize; use filecoin_hashers::Hasher; +use fr32::{write_unpadded, Fr32Reader}; use log::info; use merkletree::store::{DiskStore, LevelCacheStore, StoreConfig}; use storage_proofs::cache_key::CacheKey; @@ -24,7 +25,6 @@ use crate::constants::{ DefaultBinaryTree, DefaultOctTree, DefaultPieceDomain, DefaultPieceHasher, MINIMUM_RESERVED_BYTES_FOR_PIECE_IN_FULLY_ALIGNED_SECTOR as MINIMUM_PIECE_SIZE, }; -use crate::fr32::write_unpadded; use crate::parameters::public_params; use crate::types::{ Commitment, MerkleTreeTrait, PaddedBytesAmount, PieceInfo, PoRepConfig, PoRepProofPartitions, @@ -217,7 +217,7 @@ pub fn generate_piece_commitment( // send the source through the preprocessor let source = std::io::BufReader::new(source); - let mut fr32_reader = crate::fr32_reader::Fr32Reader::new(source); + let mut fr32_reader = Fr32Reader::new(source); let commitment = generate_piece_commitment_bytes_from_source::( &mut fr32_reader, @@ -270,7 +270,7 @@ where let written_bytes = crate::pieces::sum_piece_bytes_with_alignment(&piece_lengths); let piece_alignment = crate::pieces::get_piece_alignment(written_bytes, piece_size); - let fr32_reader = crate::fr32_reader::Fr32Reader::new(source); + let fr32_reader = Fr32Reader::new(source); // write left alignment for _ in 0..usize::from(PaddedBytesAmount::from(piece_alignment.left_bytes)) { diff --git a/filecoin-proofs/src/api/util.rs b/filecoin-proofs/src/api/util.rs index 0bc6171647..5bc9e893ba 100644 --- a/filecoin-proofs/src/api/util.rs +++ b/filecoin-proofs/src/api/util.rs @@ -1,8 +1,8 @@ use anyhow::{Context, Result}; use bellperson::bls::Fr; use filecoin_hashers::{Domain, Hasher}; +use fr32::{bytes_into_fr, fr_into_bytes}; use merkletree::merkle::{get_merkle_tree_leafs, get_merkle_tree_len}; -use storage_proofs::fr32::{bytes_into_fr, fr_into_bytes}; use storage_proofs::merkle::{get_base_tree_count, MerkleTreeTrait}; use typenum::Unsigned; diff --git a/filecoin-proofs/src/commitment_reader.rs b/filecoin-proofs/src/commitment_reader.rs index b092c7033f..e5c82d7f9b 100644 --- a/filecoin-proofs/src/commitment_reader.rs +++ b/filecoin-proofs/src/commitment_reader.rs @@ -90,13 +90,14 @@ mod tests { use crate::types::*; + use fr32::Fr32Reader; use storage_proofs::pieces::generate_piece_commitment_bytes_from_source; #[test] fn test_commitment_reader() { let piece_size = 127 * 8; let source = vec![255u8; piece_size]; - let mut fr32_reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&source)); + let mut fr32_reader = Fr32Reader::new(io::Cursor::new(&source)); let commitment1 = generate_piece_commitment_bytes_from_source::( &mut fr32_reader, @@ -104,7 +105,7 @@ mod tests { ) .expect("failed to generate piece commitment bytes from source"); - let fr32_reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&source)); + let fr32_reader = Fr32Reader::new(io::Cursor::new(&source)); let mut commitment_reader = CommitmentReader::new(fr32_reader); io::copy(&mut commitment_reader, &mut io::sink()).expect("io copy failed"); diff --git a/filecoin-proofs/src/lib.rs b/filecoin-proofs/src/lib.rs index 052eb75e9b..6e980db540 100644 --- a/filecoin-proofs/src/lib.rs +++ b/filecoin-proofs/src/lib.rs @@ -6,8 +6,6 @@ mod caches; mod commitment_reader; pub mod constants; -pub mod fr32; -pub mod fr32_reader; pub mod param; pub mod parameters; pub mod pieces; diff --git a/filecoin-proofs/src/pieces.rs b/filecoin-proofs/src/pieces.rs index 833e93203f..2be494913d 100644 --- a/filecoin-proofs/src/pieces.rs +++ b/filecoin-proofs/src/pieces.rs @@ -6,6 +6,7 @@ use std::sync::Mutex; use anyhow::{ensure, Context, Result}; use filecoin_hashers::{HashFunction, Hasher}; +use fr32::Fr32Reader; use lazy_static::lazy_static; use log::info; use storage_proofs::util::NODE_SIZE; @@ -33,7 +34,6 @@ lazy_static! { static ref COMMITMENTS: Mutex> = Mutex::new(HashMap::new()); } use crate::commitment_reader::CommitmentReader; -use crate::fr32_reader::Fr32Reader; #[derive(Debug, Clone)] pub struct EmptySource { diff --git a/filecoin-proofs/src/types/bytes_amount.rs b/filecoin-proofs/src/types/bytes_amount.rs index ae53df80ac..5a19bb00d5 100644 --- a/filecoin-proofs/src/types/bytes_amount.rs +++ b/filecoin-proofs/src/types/bytes_amount.rs @@ -1,9 +1,8 @@ use std::ops::{Add, Sub}; +use fr32::{to_padded_bytes, to_unpadded_bytes}; use serde::{Deserialize, Serialize}; -use crate::fr32::{to_padded_bytes, to_unpadded_bytes}; - pub struct PoStProofBytesAmount(pub usize); pub struct PoRepProofBytesAmount(pub usize); diff --git a/filecoin-proofs/src/types/sector_size.rs b/filecoin-proofs/src/types/sector_size.rs index 7721a7d913..daa685265d 100644 --- a/filecoin-proofs/src/types/sector_size.rs +++ b/filecoin-proofs/src/types/sector_size.rs @@ -1,4 +1,5 @@ -use crate::fr32::to_unpadded_bytes; +use fr32::to_unpadded_bytes; + use crate::types::*; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] diff --git a/filecoin-proofs/tests/mod.rs b/filecoin-proofs/tests/mod.rs index 4b2bf5d47f..917352fae7 100644 --- a/filecoin-proofs/tests/mod.rs +++ b/filecoin-proofs/tests/mod.rs @@ -1,10 +1,10 @@ use bellperson::bls::Fr; use ff::Field; +use fr32::bytes_into_fr; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs::api_version::ApiVersion; -use storage_proofs::fr32::bytes_into_fr; use storage_proofs::sector::SectorId; use filecoin_proofs::as_safe_commitment; diff --git a/fr32/Cargo.toml b/fr32/Cargo.toml new file mode 100644 index 0000000000..1519fb468b --- /dev/null +++ b/fr32/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "fr32" +version = "0.1.0" +authors = ["dignifiedquire "] +description = "Filecoin proofs Fr/32-byte conversion tooling" +license = "MIT OR Apache-2.0" +edition = "2018" +repository = "https://github.com/filecoin-project/rust-fil-proofs" + +[dependencies] +anyhow = "1.0.23" +bellperson = { version = "0.12.3", default-features = false } +byte-slice-cast = "1.0.0" +byteorder = "1" +ff = { version = "0.2.3", package = "fff" } +thiserror = "1.0.6" + +[dev-dependencies] +bitvec = "0.17" +criterion = "0.3" +itertools = "0.9" +pretty_assertions = "0.6.1" +rand = "0.7" +rand_xorshift = "0.2.0" + +[features] +default = ["pairing"] +blst = ["bellperson/blst"] +gpu = ["bellperson/gpu"] +pairing = ["bellperson/pairing"] + +[[bench]] +name = "fr" +harness = false diff --git a/storage-proofs/core/benches/fr.rs b/fr32/benches/fr.rs similarity index 91% rename from storage-proofs/core/benches/fr.rs rename to fr32/benches/fr.rs index 0f30f85543..24c0beb69e 100644 --- a/storage-proofs/core/benches/fr.rs +++ b/fr32/benches/fr.rs @@ -1,8 +1,8 @@ use bellperson::bls::Fr; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use ff::Field; +use fr32::{bytes_into_fr, fr_into_bytes}; use rand::thread_rng; -use storage_proofs_core::fr32::{bytes_into_fr, fr_into_bytes}; fn fr_benchmark(c: &mut Criterion) { c.bench_function("fr-to-bytes-32", move |b| { diff --git a/storage-proofs/core/src/fr32.rs b/fr32/src/convert.rs similarity index 52% rename from storage-proofs/core/src/fr32.rs rename to fr32/src/convert.rs index d91d98aa3e..ac058c7a18 100644 --- a/storage-proofs/core/src/fr32.rs +++ b/fr32/src/convert.rs @@ -1,59 +1,48 @@ -use crate::error::*; - -use anyhow::ensure; +#[cfg(any(feature = "pairing", feature = "blst"))] +use anyhow::Result; use bellperson::bls::{Fr, FrRepr}; -use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; +use byteorder::{ByteOrder, LittleEndian}; +use ff::PrimeField; +#[cfg(feature = "pairing")] +use ff::PrimeFieldRepr; -// Contains 32 bytes whose little-endian value represents an Fr. -// Invariants: -// - Value MUST represent a valid Fr. -// - Length must be 32. -pub type Fr32 = [u8]; +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Bytes could not be converted to Fr")] + BadFrBytes, +} -// Contains one or more 32-byte chunks whose little-endian values represent Frs. -// Invariants: -// - Value of each 32-byte chunks MUST represent valid Frs. -// - Total length must be a multiple of 32. -// That is to say: each 32-byte chunk taken alone must be a valid Fr32. +/// Contains one or more 32-byte chunks whose little-endian values represent Frs. +/// Invariants: +/// - Value of each 32-byte chunks MUST represent valid Frs. +/// - Total length must be a multiple of 32. +/// That is to say: each 32-byte chunk taken alone must be a valid Fr32. pub type Fr32Vec = Vec; -// Array whose little-endian value represents an Fr. -// Invariants: -// - Value MUST represent a valid Fr. +/// Array whose little-endian value represents an Fr. +/// Invariants: +/// - Value MUST represent a valid Fr. pub type Fr32Ary = [u8; 32]; -// Takes a slice of bytes and returns an Fr if byte slice is exactly 32 bytes and does not overflow. -// Otherwise, returns a BadFrBytesError. +/// Takes a slice of bytes and returns an Fr if byte slice is exactly 32 bytes and does not overflow. +/// Otherwise, returns a BadFrBytesError. #[cfg(feature = "pairing")] pub fn bytes_into_fr(bytes: &[u8]) -> Result { - use anyhow::Context; - use ff::{PrimeField, PrimeFieldRepr}; - + use anyhow::{ensure, Context}; ensure!(bytes.len() == 32, Error::BadFrBytes); - - let mut fr_repr = <::Repr as Default>::default(); + let mut fr_repr = FrRepr::default(); fr_repr.read_le(bytes).context(Error::BadFrBytes)?; - Fr::from_repr(fr_repr).map_err(|_| Error::BadFrBytes.into()) } #[cfg(feature = "blst")] pub fn bytes_into_fr(bytes: &[u8]) -> Result { use std::convert::TryInto; - Fr::from_bytes_le(bytes.try_into().map_err(|_| Error::BadFrBytes)?) .ok_or_else(|| Error::BadFrBytes.into()) } -#[inline] -pub fn trim_bytes_to_fr_safe(r: &[u8]) -> Result> { - ensure!(r.len() == 32, Error::BadFrBytes); - let mut res = r[..32].to_vec(); - // strip last two bits, to ensure result is in Fr. - res[31] &= 0b0011_1111; - Ok(res) -} - +/// Bytes is little-endian. #[inline] pub fn bytes_into_fr_repr_safe(r: &[u8]) -> FrRepr { debug_assert!(r.len() == 32); @@ -75,11 +64,9 @@ pub fn bytes_into_fr_repr_safe(r: &[u8]) -> FrRepr { FrRepr(repr) } -// Takes an Fr and returns a vector of exactly 32 bytes guaranteed to contain a valid Fr. +/// Takes an Fr and returns a vector of exactly 32 bytes guaranteed to contain a valid Fr. #[cfg(feature = "pairing")] pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec { - use ff::{PrimeField, PrimeFieldRepr}; - let mut out = Vec::with_capacity(32); fr.into_repr().write_le(&mut out).expect("write_le failure"); out @@ -90,37 +77,8 @@ pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec { fr.to_bytes_le().to_vec() } -// Takes a slice of bytes and returns a vector of Fr -- or an error if either bytes is not a multiple of 32 bytes -// or any 32-byte chunk overflows and does not contain a valid Fr. -pub fn bytes_into_frs(bytes: &[u8]) -> Result> { - bytes - .chunks(32) - .map(|ref chunk| bytes_into_fr(chunk)) - .collect() -} - -// Takes a slice of Frs and returns a vector of bytes, guaranteed to have a size which is a multiple of 32, -// with every 32-byte chunk representing a valid Fr. -pub fn frs_into_bytes(frs: &[Fr]) -> Fr32Vec { - frs.iter().flat_map(|fr| fr_into_bytes(fr)).collect() -} - -// Takes a u32 and returns an Fr. -pub fn u32_into_fr(n: u32) -> Fr { - let mut buf: Fr32Vec = vec![0u8; 32]; - let mut w = &mut buf[0..4]; - w.write_u32::(n).expect("write_u32 failure"); - - bytes_into_fr(&buf).expect("should never fail since u32 is in the field") -} - -// Takes a u64 and returns an Fr. pub fn u64_into_fr(n: u64) -> Fr { - let mut buf: Fr32Vec = vec![0u8; 32]; - let mut w = &mut buf[0..8]; - w.write_u64::(n).expect("write_u64 failure"); - - bytes_into_fr(&buf).expect("should never fail since u64 is in the field") + Fr::from_repr(FrRepr::from(n)).expect("failed to convert u64 into Fr (should never fail)") } #[cfg(test)] @@ -133,12 +91,12 @@ mod tests { if expect_success { let f = fr_result.expect("Failed to convert bytes to `Fr`"); let b2 = fr_into_bytes(&f); - assert_eq!(bytes.to_vec(), b2); } else { assert!(fr_result.is_err(), "expected a decoding error") } } + #[test] fn test_bytes_into_fr_into_bytes() { bytes_fr_test( @@ -181,21 +139,4 @@ mod tests { false, ); } - - fn bytes_into_frs_into_bytes_test(bytes: &Fr32) { - let frs = bytes_into_frs(bytes).expect("Failed to convert bytes into a `Vec`"); - assert!(frs.len() == 3); - let bytes_back = frs_into_bytes(&frs); - assert!(bytes.to_vec() == bytes_back); - } - - #[test] - fn test_bytes_into_frs_into_bytes() { - let bytes = b"012345678901234567890123456789--012345678901234567890123456789--012345678901234567890123456789--"; - bytes_into_frs_into_bytes_test(&bytes[..]); - - let _short_bytes = b"012345678901234567890123456789--01234567890123456789"; - // This will panic because _short_bytes is not a multiple of 32 bytes. - // bytes_into_frs_into_bytes_test(&_short_bytes[..]); - } } diff --git a/fr32/src/lib.rs b/fr32/src/lib.rs new file mode 100644 index 0000000000..b360c60425 --- /dev/null +++ b/fr32/src/lib.rs @@ -0,0 +1,7 @@ +mod convert; +mod padding; +mod reader; + +pub use convert::*; +pub use padding::*; +pub use reader::*; diff --git a/filecoin-proofs/src/fr32.rs b/fr32/src/padding.rs similarity index 87% rename from filecoin-proofs/src/fr32.rs rename to fr32/src/padding.rs index 5c5237918a..0563ebc3c8 100644 --- a/filecoin-proofs/src/fr32.rs +++ b/fr32/src/padding.rs @@ -1,8 +1,5 @@ use std::cmp::min; -use std::io::{self, Error, ErrorKind, Read, Seek, SeekFrom, Write}; - -use anyhow::{ensure, Result}; -use bitvec::{order::Lsb0 as LittleEndian, vec::BitVec}; +use std::io::{self, Error, ErrorKind, Write}; /** PaddingMap represents a mapping between data and its padded equivalent. @@ -153,7 +150,7 @@ an additional summary of what was already discussed. **/ #[derive(Debug)] -pub struct PaddingMap { +struct PaddingMap { /// The number of bits of raw data in an element. data_bits: usize, /// Number of bits in an element: `data_bits` + `pad_bits()`. Its value @@ -169,45 +166,11 @@ pub struct PaddingMap { // This is the padding map corresponding to Fr32. // Most of the code in this module is general-purpose and could move elsewhere. // The application-specific wrappers which implicitly use Fr32 embed the FR32_PADDING_MAP. -pub const FR32_PADDING_MAP: PaddingMap = PaddingMap { +const FR32_PADDING_MAP: PaddingMap = PaddingMap { data_bits: 254, element_bits: 256, }; -pub type BitVecLEu8 = BitVec; - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// Convenience interface for API functions – all bundling FR32_PADDING_MAP -// parameter/return types are tuned for current caller convenience. - -pub fn target_unpadded_bytes(target: &mut W) -> io::Result -where - W: Seek, -{ - let (_, unpadded, _) = FR32_PADDING_MAP.target_offsets(target)?; - - Ok(unpadded) -} - -// Leave the actual truncation to caller, since we can't do it generically. -// Return the length to which target should be truncated. -// We might should also handle zero-padding what will become the final byte of target. -// Technically, this should be okay though because that byte will always be overwritten later. -// If we decide this is unnecessary, then we don't need to pass target at all. -pub fn almost_truncate_to_unpadded_bytes( - _target: &mut W, - length: u64, -) -> io::Result -where - W: Read + Write + Seek, -{ - let padded = - BitByte::from_bits(FR32_PADDING_MAP.transform_bit_offset((length * 8) as usize, true)); - let real_length = padded.bytes_needed(); - let _final_bit_count = padded.bits; - Ok(real_length) -} - pub fn to_unpadded_bytes(padded_bytes: u64) -> u64 { FR32_PADDING_MAP.transform_byte_offset(padded_bytes as usize, false) as u64 } @@ -221,70 +184,33 @@ pub fn to_padded_bytes(unpadded_bytes: usize) -> usize { // with bit precision, that is, not rounded. // Invariant: it is an error for bits to be > 7. #[derive(Debug)] -pub struct BitByte { +struct BitByte { bytes: usize, bits: usize, } impl BitByte { // Create a BitByte from number of bits. Guaranteed to return a well-formed value (bits < 8) - pub fn from_bits(bits: usize) -> BitByte { + fn from_bits(bits: usize) -> BitByte { BitByte { bytes: bits / 8, bits: bits % 8, } } - pub fn from_bytes(bytes: usize) -> BitByte { - Self::from_bits(bytes * 8) - } - // How many bits in the BitByte (inverse of from_bits). - pub fn total_bits(&self) -> usize { + fn total_bits(&self) -> usize { self.bytes * 8 + self.bits } - // True if the BitByte has no bits component. - pub fn is_byte_aligned(&self) -> bool { - self.bits == 0 - } - // How many distinct bytes are needed to represent data of this size? - pub fn bytes_needed(&self) -> usize { + fn bytes_needed(&self) -> usize { self.bytes + if self.bits == 0 { 0 } else { 1 } } } impl PaddingMap { - pub fn new(data_bits: usize, element_bits: usize) -> Result { - // Check that we add less than 1 byte of padding (sub-byte padding). - ensure!( - element_bits - data_bits <= 7, - "Padding (num bits: {}) must be less than 1 byte.", - element_bits - data_bits - ); - // Check that the element is byte aligned. - ensure!( - element_bits % 8 == 0, - "Element (num bits: {}) must be byte aligned.", - element_bits - ); - - Ok(PaddingMap { - data_bits, - element_bits, - }) - } - - pub fn pad(&self, bits_out: &mut BitVecLEu8) { - for _ in 0..self.pad_bits() { - bits_out.push(false) - } - // TODO: Optimization: Drop this explicit `push` padding, the padding - // should happen implicitly when byte-aligning the data unit. - } - - pub fn pad_bits(&self) -> usize { + fn pad_bits(&self) -> usize { self.element_bits - self.data_bits } @@ -293,7 +219,7 @@ impl PaddingMap { // generated padded bit stream, that is, not byte aligned (so we // don't count the extra bits here). If `padding` is `false` calculate // the inverse transformation. - pub fn transform_bit_offset(&self, pos: usize, padding: bool) -> usize { + fn transform_bit_offset(&self, pos: usize, padding: bool) -> usize { // Set the sizes we're converting to and from. let (from_size, to_size) = if padding { (self.data_bits, self.element_bits) @@ -320,7 +246,7 @@ impl PaddingMap { // TODO: Evaluate the relationship between this function and `transform_bit_offset`, // it seems the two could be merged, or at least restructured to better expose // their differences. - pub fn transform_byte_offset(&self, pos: usize, padding: bool) -> usize { + fn transform_byte_offset(&self, pos: usize, padding: bool) -> usize { let transformed_bit_pos = self.transform_bit_offset(pos * 8, padding); let transformed_byte_pos = transformed_bit_pos as f64 / 8.; @@ -345,7 +271,7 @@ impl PaddingMap { // in bytes (since elements -with padding- are byte aligned). // - the number of bits left to read (write) from (to) the current // data unit (assuming it's full). - pub fn next_boundary(&self, position: &BitByte) -> (usize, usize) { + fn next_boundary(&self, position: &BitByte) -> (usize, usize) { let position_bits = position.total_bits(); let (_, bits_after_last_boundary) = div_rem(position_bits, self.element_bits); @@ -356,36 +282,6 @@ impl PaddingMap { (next_element_position_bits / 8, remaining_data_unit_bits) } - - // For a `Seek`able `target` of a byte-aligned padded layout, return: - // - the size in bytes - // - the size in bytes of raw data which corresponds to the `target` size - // - a BitByte representing the number of padded bits contained in the - // byte-aligned padded layout - pub fn target_offsets(&self, target: &mut W) -> io::Result<(u64, u64, BitByte)> - where - W: Seek, - { - // The current position in `target` is the number of padded bytes already written - // to the byte-aligned stream. - let padded_bytes = target.seek(SeekFrom::End(0))?; - - // Deduce the number of input raw bytes that generated that padded byte size. - let raw_data_bytes = self.transform_byte_offset(padded_bytes as usize, false); - - // With the number of raw data bytes elucidated it can now be specified the - // number of padding bits in the generated bit stream (before it was converted - // to a byte-aligned stream), that is, `raw_data_bytes * 8` is not necessarily - // `padded_bits`). - let padded_bits = self.transform_bit_offset(raw_data_bytes * 8, true); - - Ok(( - padded_bytes, - raw_data_bytes as u64, - BitByte::from_bits(padded_bits), - )) - // TODO: Why do we use `usize` internally and `u64` externally? - } } #[inline] @@ -450,7 +346,7 @@ the unique bit `0`, that just *started* at that position but doesn't necessarily carry that value.) **/ -pub fn shift_bits(input: &[u8], amount: usize, is_left: bool) -> Vec { +fn shift_bits(input: &[u8], amount: usize, is_left: bool) -> Vec { debug_assert!(amount >= 1); debug_assert!(amount <= 7); @@ -571,12 +467,7 @@ as it's no longer needed.) // may need to be moved elsewhere which would allow to reuse that term). // This also will imply removing the hardcoded `8`s (size of byte). #[inline] -pub fn extract_bits_and_shift( - input: &[u8], - pos: usize, - num_bits: usize, - new_offset: usize, -) -> Vec { +fn extract_bits_and_shift(input: &[u8], pos: usize, num_bits: usize, new_offset: usize) -> Vec { debug_assert!(input.len() * 8 >= pos + num_bits); debug_assert!(new_offset <= 7); @@ -638,14 +529,14 @@ pub fn extract_bits_and_shift( // Set to zero all the bits to the "left" of the `offset` including // it, that is, [MSB; `offset`]. #[inline] -pub fn clear_left_bits(byte: &mut u8, offset: usize) { +fn clear_left_bits(byte: &mut u8, offset: usize) { *(byte) &= (1 << offset) - 1 } // Set to zero all the bits to the "right" of the `offset` excluding // it, that is, (`offset`; LSB]. #[inline] -pub fn clear_right_bits(byte: &mut u8, offset: usize) { +fn clear_right_bits(byte: &mut u8, offset: usize) { *(byte) &= !((1 << offset) - 1) } @@ -740,7 +631,7 @@ The reader will generally operate with bit precision, even if the padded layout is byte-aligned (no extra bits) the data inside it isn't (since we pad at the bit-level). **/ -pub fn write_unpadded_aux( +fn write_unpadded_aux( padding_map: &PaddingMap, source: &[u8], target: &mut W, @@ -860,10 +751,20 @@ where mod tests { use super::*; + use std::io::Read; + + use bitvec::{order::Lsb0 as LittleEndian, vec::BitVec}; use itertools::Itertools; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; + use crate::Fr32Reader; + + const TEST_SEED: [u8; 16] = [ + 0x59, 0x62, 0xbe, 0x5d, 0x76, 0x3d, 0x31, 0x8d, 0x17, 0xdb, 0x37, 0x32, 0x54, 0x06, 0xbc, + 0xe5, + ]; + #[test] fn test_position() { let mut bits = 0; @@ -883,7 +784,7 @@ mod tests { // Length of the data vector we'll be extracting from. let len = 20; - let rng = &mut XorShiftRng::from_seed(crate::TEST_SEED); + let rng = &mut XorShiftRng::from_seed(TEST_SEED); let data: Vec = (0..len).map(|_| rng.gen()).collect(); // TODO: Evaluate designing a scattered pattered of `pos` and `num_bits` @@ -893,14 +794,14 @@ mod tests { let num_bits = rng.gen_range(1, data.len() * 8 - pos); let new_offset = rng.gen_range(0, 8); - let mut bv = BitVecLEu8::new(); + let mut bv = BitVec::::new(); bv.extend( - BitVecLEu8::from(&data[..]) + BitVec::::from(&data[..]) .into_iter() .skip(pos) .take(num_bits), ); - let shifted_bv: BitVecLEu8 = bv >> new_offset; + let shifted_bv: BitVec = bv >> new_offset; assert_eq!( shifted_bv.as_slice(), @@ -914,7 +815,7 @@ mod tests { #[test] fn test_bit_shifts() { let len = 5; - let rng = &mut XorShiftRng::from_seed(crate::TEST_SEED); + let rng = &mut XorShiftRng::from_seed(TEST_SEED); for amount in 1..8 { for left in [true, false].iter() { @@ -971,7 +872,7 @@ mod tests { let len = 1016; // Use a multiple of 254. let data = vec![255u8; len]; let mut padded = Vec::new(); - let mut reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&data)); + let mut reader = Fr32Reader::new(io::Cursor::new(&data)); reader .read_to_end(&mut padded) .expect("in-memory read failed"); @@ -993,13 +894,13 @@ mod tests { // different lengths of raw data at different offset, check integrity. #[test] fn test_read_write_padded_offset() { - let rng = &mut XorShiftRng::from_seed(crate::TEST_SEED); + let rng = &mut XorShiftRng::from_seed(TEST_SEED); let len = 1016; let data: Vec = (0..len).map(|_| rng.gen()).collect(); let mut padded = Vec::new(); - let mut reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&data)); + let mut reader = Fr32Reader::new(io::Cursor::new(&data)); reader .read_to_end(&mut padded) .expect("in-memory read failed"); diff --git a/filecoin-proofs/src/fr32_reader.rs b/fr32/src/reader.rs similarity index 98% rename from filecoin-proofs/src/fr32_reader.rs rename to fr32/src/reader.rs index 4491c5d0d2..e6756c1710 100644 --- a/filecoin-proofs/src/fr32_reader.rs +++ b/fr32/src/reader.rs @@ -1,6 +1,7 @@ -use byte_slice_cast::*; use std::io; +use byte_slice_cast::{AsByteSlice, AsSliceOf}; + /// The number of Frs per Block. const NUM_FRS_PER_BLOCK: usize = 4; /// The amount of bits in an Fr when not padded. @@ -169,9 +170,13 @@ impl io::Read for Fr32Reader { #[cfg(test)] mod tests { use super::*; - use pretty_assertions::assert_eq; + use std::io::Read; + use pretty_assertions::assert_eq; + + use crate::bytes_into_fr; + const DATA_BITS: u64 = 254; const TARGET_BITS: u64 = 256; @@ -334,7 +339,7 @@ mod tests { fn validate_fr32(bytes: &[u8]) { let chunks = (bytes.len() as f64 / 32_f64).ceil() as usize; for (i, chunk) in bytes.chunks(32).enumerate() { - let _ = storage_proofs::fr32::bytes_into_fr(chunk).unwrap_or_else(|_| { + let _ = bytes_into_fr(chunk).unwrap_or_else(|_| { panic!( "chunk {}/{} cannot be converted to valid Fr: {:?}", i + 1, diff --git a/storage-proofs/core/Cargo.toml b/storage-proofs/core/Cargo.toml index 2ae7502844..671d601550 100644 --- a/storage-proofs/core/Cargo.toml +++ b/storage-proofs/core/Cargo.toml @@ -44,6 +44,7 @@ cpu-time = { version = "1.0", optional = true } gperftools = { version = "0.2", optional = true } num_cpus = "1.10.1" semver = "0.11.0" +fr32 = { path = "../../fr32", default-features = false } [dev-dependencies] proptest = "0.10" @@ -62,9 +63,9 @@ big-sector-sizes-bench = [] measurements = ["cpu-time", "gperftools"] profile = ["measurements"] -gpu = ["bellperson/gpu", "neptune/gpu", "filecoin-hashers/gpu"] -pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde", "filecoin-hashers/pairing"] -blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde", "filecoin-hashers/blst"] +gpu = ["bellperson/gpu", "neptune/gpu", "filecoin-hashers/gpu", "fr32/gpu"] +pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde", "filecoin-hashers/pairing", "fr32/pairing"] +blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde", "filecoin-hashers/blst", "fr32/blst"] [[bench]] name = "sha256" @@ -82,10 +83,6 @@ harness = false name = "xor" harness = false -[[bench]] -name = "fr" -harness = false - [[bench]] name = "merkle" harness = false diff --git a/storage-proofs/core/src/drgraph.rs b/storage-proofs/core/src/drgraph.rs index 6a7ee6ca73..c212095e69 100644 --- a/storage-proofs/core/src/drgraph.rs +++ b/storage-proofs/core/src/drgraph.rs @@ -3,6 +3,7 @@ use std::marker::PhantomData; use anyhow::ensure; use filecoin_hashers::{Hasher, PoseidonArity}; +use fr32::bytes_into_fr_repr_safe; use generic_array::typenum; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; @@ -11,7 +12,6 @@ use sha2::{Digest, Sha256}; use crate::api_version::ApiVersion; use crate::crypto::{derive_porep_domain_seed, DRSAMPLE_DST}; use crate::error::*; -use crate::fr32::bytes_into_fr_repr_safe; use crate::parameter_cache::ParameterSetMetadata; use crate::util::{data_at_node_offset, NODE_SIZE}; use crate::PoRepID; diff --git a/storage-proofs/core/src/error.rs b/storage-proofs/core/src/error.rs index 8f7433bb9d..8be39e9bb7 100644 --- a/storage-proofs/core/src/error.rs +++ b/storage-proofs/core/src/error.rs @@ -8,8 +8,6 @@ pub use anyhow::Result; /// Custom error types #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("Bytes could not be converted to Fr")] - BadFrBytes, #[error("Could not create PieceInclusionProof (probably bad piece commitment: comm_p)")] BadPieceCommitment, #[error("Out of bounds access {} > {}", _0, _1)] diff --git a/storage-proofs/core/src/gadgets/por.rs b/storage-proofs/core/src/gadgets/por.rs index aea8c32e52..bed4a5080b 100644 --- a/storage-proofs/core/src/gadgets/por.rs +++ b/storage-proofs/core/src/gadgets/por.rs @@ -513,6 +513,7 @@ mod tests { use bellperson::gadgets::multipack; use ff::Field; + use fr32::{bytes_into_fr, fr_into_bytes}; use generic_array::typenum; use merkletree::store::VecStore; use pretty_assertions::assert_eq; @@ -520,7 +521,6 @@ mod tests { use rand_xorshift::XorShiftRng; use crate::compound_proof; - use crate::fr32::{bytes_into_fr, fr_into_bytes}; use crate::merkle::{ create_base_merkle_tree, generate_tree, get_base_tree_count, MerkleProofTrait, MerkleTreeWrapper, ResTree, diff --git a/storage-proofs/core/src/lib.rs b/storage-proofs/core/src/lib.rs index 30aa9d966b..23993a6401 100644 --- a/storage-proofs/core/src/lib.rs +++ b/storage-proofs/core/src/lib.rs @@ -14,7 +14,6 @@ pub mod crypto; pub mod data; pub mod drgraph; pub mod error; -pub mod fr32; pub mod gadgets; pub mod measurements; pub mod merkle; diff --git a/storage-proofs/core/src/pieces.rs b/storage-proofs/core/src/pieces.rs index 4b23102991..7049db4bd4 100644 --- a/storage-proofs/core/src/pieces.rs +++ b/storage-proofs/core/src/pieces.rs @@ -2,10 +2,10 @@ use std::io::Read; use anyhow::{ensure, Context}; use filecoin_hashers::{Domain, Hasher}; +use fr32::Fr32Ary; use merkletree::merkle::next_pow2; use crate::error::*; -use crate::fr32::Fr32Ary; use crate::merkle::BinaryMerkleTree; use crate::util::NODE_SIZE; diff --git a/storage-proofs/core/src/util.rs b/storage-proofs/core/src/util.rs index 1206a126e8..b3c9b53c7f 100644 --- a/storage-proofs/core/src/util.rs +++ b/storage-proofs/core/src/util.rs @@ -178,7 +178,6 @@ pub fn default_rows_to_discard(leafs: usize, arity: usize) -> usize { mod tests { use super::*; - use crate::fr32::fr_into_bytes; use bellperson::bls::*; use bellperson::gadgets::boolean::Boolean; use bellperson::gadgets::num; @@ -186,6 +185,7 @@ mod tests { use bellperson::ConstraintSystem; use ff::Field; use filecoin_hashers::{sha256::*, HashFunction}; + use fr32::fr_into_bytes; use merkletree::hash::Algorithm; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/tests/por_circuit.rs b/storage-proofs/core/tests/por_circuit.rs index 90ecbc6e87..5006fe1dd0 100644 --- a/storage-proofs/core/tests/por_circuit.rs +++ b/storage-proofs/core/tests/por_circuit.rs @@ -8,6 +8,7 @@ use ff::Field; use filecoin_hashers::{ blake2s::Blake2sHasher, poseidon::PoseidonHasher, sha256::Sha256Hasher, Domain, Hasher, }; +use fr32::{bytes_into_fr, fr_into_bytes}; use generic_array::typenum::{Unsigned, U0, U2, U4, U8}; use merkletree::store::VecStore; use pretty_assertions::assert_eq; @@ -15,7 +16,6 @@ use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ compound_proof::CompoundProof, - fr32::{bytes_into_fr, fr_into_bytes}, gadgets::por::{ challenge_into_auth_path_bits, por_no_challenge_input, PoRCircuit, PoRCompound, }, diff --git a/storage-proofs/core/tests/por_compound.rs b/storage-proofs/core/tests/por_compound.rs index 508203d3a3..8a230689d6 100644 --- a/storage-proofs/core/tests/por_compound.rs +++ b/storage-proofs/core/tests/por_compound.rs @@ -5,6 +5,7 @@ use bellperson::{ }; use ff::Field; use filecoin_hashers::{poseidon::PoseidonHasher, Hasher}; +use fr32::{bytes_into_fr, fr_into_bytes}; use generic_array::typenum::{U0, U2, U4, U8}; use merkletree::store::VecStore; use pretty_assertions::assert_eq; @@ -12,7 +13,6 @@ use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ compound_proof::{self, CompoundProof}, - fr32::{bytes_into_fr, fr_into_bytes}, gadgets::por::PoRCompound, merkle::{ create_base_merkle_tree, generate_tree, get_base_tree_count, MerkleTreeTrait, diff --git a/storage-proofs/core/tests/por_vanilla.rs b/storage-proofs/core/tests/por_vanilla.rs index 8b37b42f7f..a368b22318 100644 --- a/storage-proofs/core/tests/por_vanilla.rs +++ b/storage-proofs/core/tests/por_vanilla.rs @@ -5,13 +5,13 @@ use ff::Field; use filecoin_hashers::{ blake2s::Blake2sHasher, poseidon::PoseidonHasher, sha256::Sha256Hasher, Domain, Hasher, }; +use fr32::fr_into_bytes; use generic_array::typenum::{U0, U2, U4}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ api_version::ApiVersion, drgraph::{BucketGraph, Graph, BASE_DEGREE}, - fr32::fr_into_bytes, merkle::{create_base_merkle_tree, DiskStore, MerkleTreeTrait, MerkleTreeWrapper}, por::{self, PoR}, proof::ProofScheme, diff --git a/storage-proofs/porep/Cargo.toml b/storage-proofs/porep/Cargo.toml index 2c15db7e31..38142b0327 100644 --- a/storage-proofs/porep/Cargo.toml +++ b/storage-proofs/porep/Cargo.toml @@ -39,6 +39,7 @@ byte-slice-cast = "1.0.0" hwloc = "0.3.0" libc = "0.2" fdlimit = "0.2.0" +fr32 = { path = "../../fr32", default-features = false } [dev-dependencies] tempfile = "3" @@ -50,9 +51,9 @@ filecoin-hashers = { path = "../../filecoin-hashers", version = "1.0.0", default [features] default = ["pairing", "gpu"] -gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/gpu", "bellperson/gpu"] -pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing"] -blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst"] +gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/gpu", "bellperson/gpu", "fr32/gpu"] +pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing", "fr32/pairing"] +blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst", "fr32/blst"] single-threaded = [] [[bench]] diff --git a/storage-proofs/porep/benches/encode.rs b/storage-proofs/porep/benches/encode.rs index 47ab157692..7f5ad346b9 100644 --- a/storage-proofs/porep/benches/encode.rs +++ b/storage-proofs/porep/benches/encode.rs @@ -5,9 +5,9 @@ use filecoin_hashers::{ sha256::Sha256Hasher, {Domain, Hasher}, }; +use fr32::fr_into_bytes; use rand::thread_rng; use storage_proofs_core::api_version::ApiVersion; -use storage_proofs_core::fr32::fr_into_bytes; use storage_proofs_porep::stacked::{ create_label::single::{create_label, create_label_exp}, StackedBucketGraph, diff --git a/storage-proofs/porep/src/drg/vanilla.rs b/storage-proofs/porep/src/drg/vanilla.rs index ee7a22fe8d..53f5e96cb1 100644 --- a/storage-proofs/porep/src/drg/vanilla.rs +++ b/storage-proofs/porep/src/drg/vanilla.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::{ensure, Context}; use filecoin_hashers::{Domain, HashFunction, Hasher, PoseidonArity}; +use fr32::bytes_into_fr_repr_safe; use generic_array::typenum; use merkletree::store::{ReplicaConfig, StoreConfig}; use rayon::prelude::*; @@ -14,7 +15,6 @@ use storage_proofs_core::{ crypto::sloth, drgraph::Graph, error::Result, - fr32::bytes_into_fr_repr_safe, merkle::{ create_base_lcmerkle_tree, create_base_merkle_tree, BinaryLCMerkleTree, BinaryMerkleTree, LCMerkleTree, MerkleProof, MerkleProofTrait, MerkleTreeTrait, diff --git a/storage-proofs/porep/src/stacked/circuit/create_label.rs b/storage-proofs/porep/src/stacked/circuit/create_label.rs index 90fb09ccfc..f26ac47ed7 100644 --- a/storage-proofs/porep/src/stacked/circuit/create_label.rs +++ b/storage-proofs/porep/src/stacked/circuit/create_label.rs @@ -74,12 +74,12 @@ mod tests { use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; use filecoin_hashers::sha256::Sha256Hasher; + use fr32::{bytes_into_fr, fr_into_bytes}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ api_version::ApiVersion, drgraph::{Graph, BASE_DEGREE}, - fr32::{bytes_into_fr, fr_into_bytes}, util::{bytes_into_boolean_vec_be, data_at_node, NODE_SIZE}, TEST_SEED, }; diff --git a/storage-proofs/porep/src/stacked/circuit/proof.rs b/storage-proofs/porep/src/stacked/circuit/proof.rs index 31044d5f9a..c4db7f3f9c 100644 --- a/storage-proofs/porep/src/stacked/circuit/proof.rs +++ b/storage-proofs/porep/src/stacked/circuit/proof.rs @@ -5,11 +5,11 @@ use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use filecoin_hashers::{HashFunction, Hasher}; +use fr32::u64_into_fr; use storage_proofs_core::{ compound_proof::{CircuitComponent, CompoundProof}, drgraph::Graph, error::Result, - fr32::u64_into_fr, gadgets::constraint, gadgets::por::PoRCompound, merkle::{BinaryMerkleTree, MerkleTreeTrait}, diff --git a/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs b/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs index 804fb9d4e2..9fa4fb3146 100644 --- a/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs @@ -3,9 +3,9 @@ use std::marker::PhantomData; use bellperson::bls::Fr; use filecoin_hashers::Hasher; +use fr32::bytes_into_fr_repr_safe; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use storage_proofs_core::fr32::bytes_into_fr_repr_safe; use crate::encode::encode; diff --git a/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs b/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs index 7ce43adbcd..4d0fcf7af2 100644 --- a/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs @@ -1,10 +1,10 @@ use std::marker::PhantomData; use filecoin_hashers::Hasher; +use fr32::bytes_into_fr_repr_safe; use log::trace; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use storage_proofs_core::fr32::bytes_into_fr_repr_safe; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LabelingProof { diff --git a/storage-proofs/porep/src/stacked/vanilla/params.rs b/storage-proofs/porep/src/stacked/vanilla/params.rs index 607c419d7b..a5d7a23ad0 100644 --- a/storage-proofs/porep/src/stacked/vanilla/params.rs +++ b/storage-proofs/porep/src/stacked/vanilla/params.rs @@ -4,14 +4,15 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use filecoin_hashers::{Domain, Hasher}; +use fr32::bytes_into_fr_repr_safe; use generic_array::typenum::{self, Unsigned}; use log::trace; use merkletree::merkle::get_merkle_tree_leafs; use merkletree::store::{DiskStore, Store, StoreConfig}; use serde::{Deserialize, Serialize}; use storage_proofs_core::{ - api_version::ApiVersion, drgraph::Graph, error::Result, fr32::bytes_into_fr_repr_safe, - merkle::*, parameter_cache::ParameterSetMetadata, util::data_at_node, + api_version::ApiVersion, drgraph::Graph, error::Result, merkle::*, + parameter_cache::ParameterSetMetadata, util::data_at_node, }; use super::{ diff --git a/storage-proofs/porep/src/stacked/vanilla/proof.rs b/storage-proofs/porep/src/stacked/vanilla/proof.rs index 30e52250fe..04f42d0745 100644 --- a/storage-proofs/porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/proof.rs @@ -434,12 +434,12 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr { use bellperson::bls::Fr; use ff::Field; + use fr32::fr_into_bytes; use generic_array::{sequence::GenericSequence, GenericArray}; use merkletree::store::DiskStore; use neptune::batch_hasher::BatcherType; use neptune::column_tree_builder::{ColumnTreeBuilder, ColumnTreeBuilderTrait}; use std::sync::{mpsc, Arc, RwLock}; - use storage_proofs_core::fr32::fr_into_bytes; info!("generating tree c using the GPU"); // Build the tree for CommC @@ -788,13 +788,13 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr TreeArity: PoseidonArity, { use bellperson::bls::Fr; + use fr32::fr_into_bytes; use merkletree::merkle::{get_merkle_tree_cache_size, get_merkle_tree_leafs}; use neptune::batch_hasher::BatcherType; use neptune::tree_builder::{TreeBuilder, TreeBuilderTrait}; use std::fs::OpenOptions; use std::io::Write; use std::sync::mpsc; - use storage_proofs_core::fr32::fr_into_bytes; let (configs, replica_config) = split_config_and_replica( tree_r_last_config.clone(), @@ -1299,12 +1299,12 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr { use bellperson::bls::Fr; use ff::Field; + use fr32::fr_into_bytes; use merkletree::merkle::{get_merkle_tree_cache_size, get_merkle_tree_leafs}; use neptune::batch_hasher::BatcherType; use neptune::tree_builder::{TreeBuilder, TreeBuilderTrait}; use std::fs::OpenOptions; use std::io::Write; - use storage_proofs_core::fr32::fr_into_bytes; let (configs, replica_config) = split_config_and_replica( tree_r_last_config.clone(), diff --git a/storage-proofs/porep/tests/drg_circuit.rs b/storage-proofs/porep/tests/drg_circuit.rs index f4b20affae..7ff9265012 100644 --- a/storage-proofs/porep/tests/drg_circuit.rs +++ b/storage-proofs/porep/tests/drg_circuit.rs @@ -5,6 +5,7 @@ use bellperson::{ }; use ff::Field; use filecoin_hashers::poseidon::PoseidonHasher; +use fr32::{bytes_into_fr, fr_into_bytes}; use generic_array::typenum::U2; use merkletree::store::StoreConfig; use pretty_assertions::assert_eq; @@ -15,7 +16,6 @@ use storage_proofs_core::{ cache_key::CacheKey, compound_proof, drgraph::{graph_height, BucketGraph, BASE_DEGREE}, - fr32::{bytes_into_fr, fr_into_bytes}, gadgets::variables::Root, merkle::MerkleProofTrait, proof::ProofScheme, diff --git a/storage-proofs/porep/tests/drg_compound.rs b/storage-proofs/porep/tests/drg_compound.rs index 4f16c23caa..d6b328f134 100644 --- a/storage-proofs/porep/tests/drg_compound.rs +++ b/storage-proofs/porep/tests/drg_compound.rs @@ -5,6 +5,7 @@ use bellperson::{ }; use ff::Field; use filecoin_hashers::{poseidon::PoseidonHasher, Hasher}; +use fr32::fr_into_bytes; use merkletree::store::StoreConfig; use pretty_assertions::assert_eq; use rand::SeedableRng; @@ -14,7 +15,6 @@ use storage_proofs_core::{ cache_key::CacheKey, compound_proof::{self, CompoundProof}, drgraph::{BucketGraph, BASE_DEGREE}, - fr32::fr_into_bytes, merkle::{BinaryMerkleTree, MerkleTreeTrait}, proof::NoRequirements, test_helper::setup_replica, diff --git a/storage-proofs/porep/tests/drg_vanilla.rs b/storage-proofs/porep/tests/drg_vanilla.rs index 14cd06a429..925d3ab2d0 100644 --- a/storage-proofs/porep/tests/drg_vanilla.rs +++ b/storage-proofs/porep/tests/drg_vanilla.rs @@ -2,6 +2,7 @@ use bellperson::bls::Fr; use ff::Field; use filecoin_hashers::{blake2s::Blake2sHasher, sha256::Sha256Hasher, Domain, Hasher}; use merkletree::store::StoreConfig; +use fr32::fr_into_bytes; use pretty_assertions::assert_eq; use rand::SeedableRng; use rand_xorshift::XorShiftRng; @@ -9,7 +10,6 @@ use storage_proofs_core::{ api_version::ApiVersion, cache_key::CacheKey, drgraph::{BucketGraph, BASE_DEGREE}, - fr32::fr_into_bytes, merkle::{BinaryMerkleTree, MerkleTreeTrait}, proof::ProofScheme, table_tests, diff --git a/storage-proofs/porep/tests/stacked_circuit.rs b/storage-proofs/porep/tests/stacked_circuit.rs index 10fe3b7f14..96556b18e0 100644 --- a/storage-proofs/porep/tests/stacked_circuit.rs +++ b/storage-proofs/porep/tests/stacked_circuit.rs @@ -5,6 +5,7 @@ use bellperson::{ }; use ff::Field; use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher, Hasher}; +use fr32::fr_into_bytes; use generic_array::typenum::{U0, U2, U4, U8}; use merkletree::store::StoreConfig; use rand::{Rng, SeedableRng}; @@ -14,7 +15,6 @@ use storage_proofs_core::{ cache_key::CacheKey, compound_proof::CompoundProof, drgraph::BASE_DEGREE, - fr32::fr_into_bytes, merkle::{get_base_tree_count, DiskTree, MerkleTreeTrait}, proof::ProofScheme, test_helper::setup_replica, diff --git a/storage-proofs/porep/tests/stacked_compound.rs b/storage-proofs/porep/tests/stacked_compound.rs index 38eb9f64f4..c8c39f7e65 100644 --- a/storage-proofs/porep/tests/stacked_compound.rs +++ b/storage-proofs/porep/tests/stacked_compound.rs @@ -5,6 +5,7 @@ use bellperson::{ }; use ff::Field; use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher, Hasher}; +use fr32::fr_into_bytes; use generic_array::typenum::{U0, U2, U4, U8}; use merkletree::store::StoreConfig; use rand::{Rng, SeedableRng}; @@ -14,7 +15,6 @@ use storage_proofs_core::{ cache_key::CacheKey, compound_proof::{self, CompoundProof}, drgraph::BASE_DEGREE, - fr32::fr_into_bytes, merkle::{get_base_tree_count, DiskTree, MerkleTreeTrait}, test_helper::setup_replica, util::default_rows_to_discard, diff --git a/storage-proofs/porep/tests/stacked_vanilla.rs b/storage-proofs/porep/tests/stacked_vanilla.rs index 9550b0cd5c..562558690e 100644 --- a/storage-proofs/porep/tests/stacked_vanilla.rs +++ b/storage-proofs/porep/tests/stacked_vanilla.rs @@ -3,6 +3,7 @@ use ff::{Field, PrimeField}; use filecoin_hashers::{ blake2s::Blake2sHasher, poseidon::PoseidonHasher, sha256::Sha256Hasher, Domain, Hasher, }; +use fr32::fr_into_bytes; use generic_array::typenum::{U0, U2, U4, U8}; use merkletree::store::{Store, StoreConfig}; use rand::{Rng, SeedableRng}; @@ -11,7 +12,6 @@ use storage_proofs_core::{ api_version::ApiVersion, cache_key::CacheKey, drgraph::BASE_DEGREE, - fr32::fr_into_bytes, merkle::{get_base_tree_count, DiskTree, MerkleTreeTrait}, proof::ProofScheme, table_tests, diff --git a/storage-proofs/post/Cargo.toml b/storage-proofs/post/Cargo.toml index fc85125da5..799e94fc68 100644 --- a/storage-proofs/post/Cargo.toml +++ b/storage-proofs/post/Cargo.toml @@ -28,6 +28,7 @@ generic-array = "0.14.4" anyhow = "1.0.23" neptune = { version = "2.2.0", default-features = false } num_cpus = "1.10.1" +fr32 = { path = "../../fr32", default-features = false } [dev-dependencies] tempfile = "3" @@ -37,6 +38,6 @@ filecoin-hashers = { path = "../../filecoin-hashers", version = "1.0.0", default [features] default = ["pairing", "gpu"] -gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu"] -pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing"] -blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst"] +gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "fr32/gpu"] +pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing", "fr32/pairing"] +blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst", "fr32/blst"] diff --git a/storage-proofs/post/src/election/vanilla.rs b/storage-proofs/post/src/election/vanilla.rs index d2f4fae2a5..a67b1c5e6b 100644 --- a/storage-proofs/post/src/election/vanilla.rs +++ b/storage-proofs/post/src/election/vanilla.rs @@ -9,6 +9,7 @@ use filecoin_hashers::{ poseidon::{PoseidonDomain, PoseidonFunction}, Domain, HashFunction, Hasher, PoseidonMDArity, }; +use fr32::fr_into_bytes; use generic_array::typenum; use log::trace; use rayon::prelude::*; @@ -18,7 +19,6 @@ use typenum::Unsigned; use storage_proofs_core::{ error::{Error, Result}, - fr32::fr_into_bytes, measurements::{measure_op, Operation}, merkle::{MerkleProof, MerkleProofTrait, MerkleTreeTrait, MerkleTreeWrapper}, parameter_cache::ParameterSetMetadata,