Merge pull request #1387 from filecoin-project/fr32-subcrate

.circleci/config.yml

@@ -622,3 +622,15 @@ workflows:
             branches:
               only:
                 - master
+
+      - test:
+          name: test_fr32
+          crate: "fr32"
+          requires:
+            - cargo_fetch
+
+      - test_blst:
+          name: test_blst_fr32
+          crate: "fr32"
+          requires:
+            - cargo_fetch

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "storage-proofs/post",
   "fil-proofs-tooling",
   "fil-proofs-param",
+  "fr32",
   "sha2raw",
   "filecoin-hashers",
 ]

filecoin-proofs/Cargo.toml

@@ -37,6 +37,7 @@ gperftools = { version = "0.2", optional = true }
 generic-array = "0.14.4"
 groupy = "0.3.0"
 byte-slice-cast = "1.0.0"
+fr32 = { path = "../fr32", default-features = false }
 
 [dev-dependencies]
 criterion = "0.3"
@@ -51,9 +52,9 @@ cpu-profile = ["gperftools"]
 heap-profile = ["gperftools/heap"]
 simd = ["storage-proofs/simd"]
 asm = ["storage-proofs/asm"]
-gpu = ["storage-proofs/gpu", "bellperson/gpu", "filecoin-hashers/gpu"]
-pairing = ["storage-proofs/pairing", "bellperson/pairing", "filecoin-hashers/pairing"]
-blst = ["storage-proofs/blst", "bellperson/blst", "filecoin-hashers/blst"]
+gpu = ["storage-proofs/gpu", "bellperson/gpu", "filecoin-hashers/gpu", "fr32/gpu"]
+pairing = ["storage-proofs/pairing", "bellperson/pairing", "filecoin-hashers/pairing", "fr32/pairing"]
+blst = ["storage-proofs/blst", "bellperson/blst", "filecoin-hashers/blst", "fr32/blst"]
 
 [[bench]]
 name = "preprocessing"

filecoin-proofs/benches/preprocessing.rs

@@ -2,7 +2,8 @@ use std::io::{self, Read};
 use std::time::Duration;
 
 use criterion::{criterion_group, criterion_main, Criterion, ParameterizedBenchmark, Throughput};
-use filecoin_proofs::{add_piece, fr32_reader::Fr32Reader, PaddedBytesAmount, UnpaddedBytesAmount};
+use filecoin_proofs::{add_piece, PaddedBytesAmount, UnpaddedBytesAmount};
+use fr32::Fr32Reader;
 use rand::{thread_rng, Rng};
 
 #[cfg(feature = "cpu-profile")]

filecoin-proofs/src/api/mod.rs

@@ -5,6 +5,7 @@ use std::path::{Path, PathBuf};
 use anyhow::{ensure, Context, Result};
 use bincode::deserialize;
 use filecoin_hashers::Hasher;
+use fr32::{write_unpadded, Fr32Reader};
 use log::info;
 use merkletree::store::{DiskStore, LevelCacheStore, StoreConfig};
 use storage_proofs::cache_key::CacheKey;
@@ -24,7 +25,6 @@ use crate::constants::{
     DefaultBinaryTree, DefaultOctTree, DefaultPieceDomain, DefaultPieceHasher,
     MINIMUM_RESERVED_BYTES_FOR_PIECE_IN_FULLY_ALIGNED_SECTOR as MINIMUM_PIECE_SIZE,
 };
-use crate::fr32::write_unpadded;
 use crate::parameters::public_params;
 use crate::types::{
     Commitment, MerkleTreeTrait, PaddedBytesAmount, PieceInfo, PoRepConfig, PoRepProofPartitions,
@@ -217,7 +217,7 @@ pub fn generate_piece_commitment<T: std::io::Read>(
 
         // send the source through the preprocessor
         let source = std::io::BufReader::new(source);
-        let mut fr32_reader = crate::fr32_reader::Fr32Reader::new(source);
+        let mut fr32_reader = Fr32Reader::new(source);
 
         let commitment = generate_piece_commitment_bytes_from_source::<DefaultPieceHasher>(
             &mut fr32_reader,
@@ -270,7 +270,7 @@ where
 
         let written_bytes = crate::pieces::sum_piece_bytes_with_alignment(&piece_lengths);
         let piece_alignment = crate::pieces::get_piece_alignment(written_bytes, piece_size);
-        let fr32_reader = crate::fr32_reader::Fr32Reader::new(source);
+        let fr32_reader = Fr32Reader::new(source);
 
         // write left alignment
         for _ in 0..usize::from(PaddedBytesAmount::from(piece_alignment.left_bytes)) {

filecoin-proofs/src/api/util.rs

@@ -1,8 +1,8 @@
 use anyhow::{Context, Result};
 use bellperson::bls::Fr;
 use filecoin_hashers::{Domain, Hasher};
+use fr32::{bytes_into_fr, fr_into_bytes};
 use merkletree::merkle::{get_merkle_tree_leafs, get_merkle_tree_len};
-use storage_proofs::fr32::{bytes_into_fr, fr_into_bytes};
 use storage_proofs::merkle::{get_base_tree_count, MerkleTreeTrait};
 use typenum::Unsigned;
 

filecoin-proofs/src/commitment_reader.rs

@@ -90,21 +90,22 @@ mod tests {
 
     use crate::types::*;
 
+    use fr32::Fr32Reader;
     use storage_proofs::pieces::generate_piece_commitment_bytes_from_source;
 
     #[test]
     fn test_commitment_reader() {
         let piece_size = 127 * 8;
         let source = vec![255u8; piece_size];
-        let mut fr32_reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&source));
+        let mut fr32_reader = Fr32Reader::new(io::Cursor::new(&source));
 
         let commitment1 = generate_piece_commitment_bytes_from_source::<DefaultPieceHasher>(
             &mut fr32_reader,
             PaddedBytesAmount::from(UnpaddedBytesAmount(piece_size as u64)).into(),
         )
         .expect("failed to generate piece commitment bytes from source");
 
-        let fr32_reader = crate::fr32_reader::Fr32Reader::new(io::Cursor::new(&source));
+        let fr32_reader = Fr32Reader::new(io::Cursor::new(&source));
         let mut commitment_reader = CommitmentReader::new(fr32_reader);
         io::copy(&mut commitment_reader, &mut io::sink()).expect("io copy failed");
 

filecoin-proofs/src/lib.rs

@@ -6,8 +6,6 @@ mod caches;
 mod commitment_reader;
 
 pub mod constants;
-pub mod fr32;
-pub mod fr32_reader;
 pub mod param;
 pub mod parameters;
 pub mod pieces;

filecoin-proofs/src/pieces.rs

@@ -6,6 +6,7 @@ use std::sync::Mutex;
 
 use anyhow::{ensure, Context, Result};
 use filecoin_hashers::{HashFunction, Hasher};
+use fr32::Fr32Reader;
 use lazy_static::lazy_static;
 use log::info;
 use storage_proofs::util::NODE_SIZE;
@@ -33,7 +34,6 @@ lazy_static! {
     static ref COMMITMENTS: Mutex<HashMap<SectorSize, Commitment>> = Mutex::new(HashMap::new());
 }
 use crate::commitment_reader::CommitmentReader;
-use crate::fr32_reader::Fr32Reader;
 
 #[derive(Debug, Clone)]
 pub struct EmptySource {

filecoin-proofs/src/types/bytes_amount.rs

@@ -1,9 +1,8 @@
 use std::ops::{Add, Sub};
 
+use fr32::{to_padded_bytes, to_unpadded_bytes};
 use serde::{Deserialize, Serialize};
 
-use crate::fr32::{to_padded_bytes, to_unpadded_bytes};
-
 pub struct PoStProofBytesAmount(pub usize);
 
 pub struct PoRepProofBytesAmount(pub usize);

filecoin-proofs/src/types/sector_size.rs

@@ -1,4 +1,5 @@
-use crate::fr32::to_unpadded_bytes;
+use fr32::to_unpadded_bytes;
+
 use crate::types::*;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]

filecoin-proofs/tests/mod.rs

@@ -1,10 +1,10 @@
 use bellperson::bls::Fr;
 use ff::Field;
+use fr32::bytes_into_fr;
 use rand::SeedableRng;
 use rand_xorshift::XorShiftRng;
 
 use storage_proofs::api_version::ApiVersion;
-use storage_proofs::fr32::bytes_into_fr;
 use storage_proofs::sector::SectorId;
 
 use filecoin_proofs::as_safe_commitment;

fr32/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "fr32"
+version = "0.1.0"
+authors = ["dignifiedquire <me@dignifiedquire.com>"]
+description = "Filecoin proofs Fr/32-byte conversion tooling"
+license = "MIT OR Apache-2.0"
+edition = "2018"
+repository = "https://github.com/filecoin-project/rust-fil-proofs"
+
+[dependencies]
+anyhow = "1.0.23"
+bellperson = { version = "0.12.3", default-features = false }
+byte-slice-cast = "1.0.0"
+byteorder = "1"
+ff = { version = "0.2.3", package = "fff" }
+thiserror = "1.0.6"
+
+[dev-dependencies]
+bitvec = "0.17"
+criterion = "0.3"
+itertools = "0.9"
+pretty_assertions = "0.6.1"
+rand = "0.7"
+rand_xorshift = "0.2.0"
+
+[features]
+default = ["pairing"]
+blst = ["bellperson/blst"]
+gpu = ["bellperson/gpu"]
+pairing = ["bellperson/pairing"]
+
+[[bench]]
+name = "fr"
+harness = false

storage-proofs/core/benches/fr.rs → fr32/benches/fr.rs

@@ -1,8 +1,8 @@
 use bellperson::bls::Fr;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use ff::Field;
+use fr32::{bytes_into_fr, fr_into_bytes};
 use rand::thread_rng;
-use storage_proofs_core::fr32::{bytes_into_fr, fr_into_bytes};
 
 fn fr_benchmark(c: &mut Criterion) {
     c.bench_function("fr-to-bytes-32", move |b| {

storage-proofs/core/src/fr32.rs → fr32/src/convert.rs

@@ -1,59 +1,48 @@
-use crate::error::*;
-
-use anyhow::ensure;
+#[cfg(any(feature = "pairing", feature = "blst"))]
+use anyhow::Result;
 use bellperson::bls::{Fr, FrRepr};
-use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
+use byteorder::{ByteOrder, LittleEndian};
+use ff::PrimeField;
+#[cfg(feature = "pairing")]
+use ff::PrimeFieldRepr;
 
-// Contains 32 bytes whose little-endian value represents an Fr.
-// Invariants:
-// - Value MUST represent a valid Fr.
-// - Length must be 32.
-pub type Fr32 = [u8];
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Bytes could not be converted to Fr")]
+    BadFrBytes,
+}
 
-// Contains one or more 32-byte chunks whose little-endian values represent Frs.
-// Invariants:
-// - Value of each 32-byte chunks MUST represent valid Frs.
-// - Total length must be a multiple of 32.
-// That is to say: each 32-byte chunk taken alone must be a valid Fr32.
+/// Contains one or more 32-byte chunks whose little-endian values represent Frs.
+/// Invariants:
+/// - Value of each 32-byte chunks MUST represent valid Frs.
+/// - Total length must be a multiple of 32.
+/// That is to say: each 32-byte chunk taken alone must be a valid Fr32.
 pub type Fr32Vec = Vec<u8>;
 
-// Array whose little-endian value represents an Fr.
-// Invariants:
-// - Value MUST represent a valid Fr.
+/// Array whose little-endian value represents an Fr.
+/// Invariants:
+/// - Value MUST represent a valid Fr.
 pub type Fr32Ary = [u8; 32];
 
-// Takes a slice of bytes and returns an Fr if byte slice is exactly 32 bytes and does not overflow.
-// Otherwise, returns a BadFrBytesError.
+/// Takes a slice of bytes and returns an Fr if byte slice is exactly 32 bytes and does not overflow.
+/// Otherwise, returns a BadFrBytesError.
 #[cfg(feature = "pairing")]
 pub fn bytes_into_fr(bytes: &[u8]) -> Result<Fr> {
-    use anyhow::Context;
-    use ff::{PrimeField, PrimeFieldRepr};
-
+    use anyhow::{ensure, Context};
     ensure!(bytes.len() == 32, Error::BadFrBytes);
-
-    let mut fr_repr = <<Fr as PrimeField>::Repr as Default>::default();
+    let mut fr_repr = FrRepr::default();
     fr_repr.read_le(bytes).context(Error::BadFrBytes)?;
-
     Fr::from_repr(fr_repr).map_err(|_| Error::BadFrBytes.into())
 }
 
 #[cfg(feature = "blst")]
 pub fn bytes_into_fr(bytes: &[u8]) -> Result<Fr> {
     use std::convert::TryInto;
-
     Fr::from_bytes_le(bytes.try_into().map_err(|_| Error::BadFrBytes)?)
         .ok_or_else(|| Error::BadFrBytes.into())
 }
 
-#[inline]
-pub fn trim_bytes_to_fr_safe(r: &[u8]) -> Result<Vec<u8>> {
-    ensure!(r.len() == 32, Error::BadFrBytes);
-    let mut res = r[..32].to_vec();
-    // strip last two bits, to ensure result is in Fr.
-    res[31] &= 0b0011_1111;
-    Ok(res)
-}
-
+/// Bytes is little-endian.
 #[inline]
 pub fn bytes_into_fr_repr_safe(r: &[u8]) -> FrRepr {
     debug_assert!(r.len() == 32);
@@ -75,11 +64,9 @@ pub fn bytes_into_fr_repr_safe(r: &[u8]) -> FrRepr {
     FrRepr(repr)
 }
 
-// Takes an Fr and returns a vector of exactly 32 bytes guaranteed to contain a valid Fr.
+/// Takes an Fr and returns a vector of exactly 32 bytes guaranteed to contain a valid Fr.
 #[cfg(feature = "pairing")]
 pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec {
-    use ff::{PrimeField, PrimeFieldRepr};
-
     let mut out = Vec::with_capacity(32);
     fr.into_repr().write_le(&mut out).expect("write_le failure");
     out
@@ -90,37 +77,8 @@ pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec {
     fr.to_bytes_le().to_vec()
 }
 
-// Takes a slice of bytes and returns a vector of Fr -- or an error if either bytes is not a multiple of 32 bytes
-// or any 32-byte chunk overflows and does not contain a valid Fr.
-pub fn bytes_into_frs(bytes: &[u8]) -> Result<Vec<Fr>> {
-    bytes
-        .chunks(32)
-        .map(|ref chunk| bytes_into_fr(chunk))
-        .collect()
-}
-
-// Takes a slice of Frs and returns a vector of bytes, guaranteed to have a size which is a multiple of 32,
-// with every 32-byte chunk representing a valid Fr.
-pub fn frs_into_bytes(frs: &[Fr]) -> Fr32Vec {
-    frs.iter().flat_map(|fr| fr_into_bytes(fr)).collect()
-}
-
-// Takes a u32 and returns an Fr.
-pub fn u32_into_fr(n: u32) -> Fr {
-    let mut buf: Fr32Vec = vec![0u8; 32];
-    let mut w = &mut buf[0..4];
-    w.write_u32::<LittleEndian>(n).expect("write_u32 failure");
-
-    bytes_into_fr(&buf).expect("should never fail since u32 is in the field")
-}
-
-// Takes a u64 and returns an Fr.
 pub fn u64_into_fr(n: u64) -> Fr {
-    let mut buf: Fr32Vec = vec![0u8; 32];
-    let mut w = &mut buf[0..8];
-    w.write_u64::<LittleEndian>(n).expect("write_u64 failure");
-
-    bytes_into_fr(&buf).expect("should never fail since u64 is in the field")
+    Fr::from_repr(FrRepr::from(n)).expect("failed to convert u64 into Fr (should never fail)")
 }
 
 #[cfg(test)]
@@ -133,12 +91,12 @@ mod tests {
         if expect_success {
             let f = fr_result.expect("Failed to convert bytes to `Fr`");
             let b2 = fr_into_bytes(&f);
-
             assert_eq!(bytes.to_vec(), b2);
         } else {
             assert!(fr_result.is_err(), "expected a decoding error")
         }
     }
+
     #[test]
     fn test_bytes_into_fr_into_bytes() {
         bytes_fr_test(
@@ -181,21 +139,4 @@ mod tests {
             false,
         );
     }
-
-    fn bytes_into_frs_into_bytes_test(bytes: &Fr32) {
-        let frs = bytes_into_frs(bytes).expect("Failed to convert bytes into a `Vec<Fr>`");
-        assert!(frs.len() == 3);
-        let bytes_back = frs_into_bytes(&frs);
-        assert!(bytes.to_vec() == bytes_back);
-    }
-
-    #[test]
-    fn test_bytes_into_frs_into_bytes() {
-        let bytes = b"012345678901234567890123456789--012345678901234567890123456789--012345678901234567890123456789--";
-        bytes_into_frs_into_bytes_test(&bytes[..]);
-
-        let _short_bytes = b"012345678901234567890123456789--01234567890123456789";
-        // This will panic because _short_bytes is not a multiple of 32 bytes.
-        // bytes_into_frs_into_bytes_test(&_short_bytes[..]);
-    }
 }