From 316051f61116e867413c403a077abc6271521cf5 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 7 Jul 2020 23:26:38 +0200 Subject: [PATCH 1/2] feat: allow for compilation on aarch64 more fixes sha2raw: update deps and enable asm by default patch ff fix compile time imports missing import move prefetch into macro fixup fil-proofs-tooling refactoring refactoring --- Cargo.toml | 4 ++ README.md | 13 +++++ fil-proofs-tooling/Cargo.toml | 5 +- fil-proofs-tooling/src/metadata.rs | 47 ++++++++++++------- filecoin-proofs/Cargo.toml | 2 +- sha2raw/Cargo.toml | 12 +++-- sha2raw/src/platform.rs | 12 ++--- storage-proofs/core/Cargo.toml | 2 +- storage-proofs/core/src/crypto/mod.rs | 2 +- storage-proofs/core/src/crypto/pedersen.rs | 22 ++------- storage-proofs/core/src/drgraph.rs | 6 +-- storage-proofs/core/src/hasher/pedersen.rs | 45 +++++++++--------- storage-proofs/core/src/hasher/sha256.rs | 6 +-- storage-proofs/core/src/parameter_cache.rs | 4 +- storage-proofs/porep/Cargo.toml | 2 +- storage-proofs/porep/src/drg/vanilla.rs | 12 ++--- .../porep/src/stacked/vanilla/cache.rs | 10 ++-- .../porep/src/stacked/vanilla/challenges.rs | 2 +- .../porep/src/stacked/vanilla/create_label.rs | 13 +---- .../src/stacked/vanilla/encoding_proof.rs | 6 +-- .../porep/src/stacked/vanilla/graph.rs | 9 +--- .../src/stacked/vanilla/labeling_proof.rs | 6 +-- .../porep/src/stacked/vanilla/macros.rs | 14 ++++++ .../porep/src/stacked/vanilla/params.rs | 2 +- storage-proofs/post/Cargo.toml | 2 +- storage-proofs/post/src/election/vanilla.rs | 18 +++---- storage-proofs/post/src/fallback/vanilla.rs | 20 ++++---- 27 files changed, 155 insertions(+), 143 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0a0b6acae..94beff80d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,7 @@ members = [ "fil-proofs-tooling", "sha2raw" ] + +[patch.crates-io] +fil-sapling-crypto = { git = "https://github.com/filecoin-project/sapling-crypto", branch = "fix/compile-nonx86" } +fff = { git = "https://github.com/filecoin-project/ff", branch = "fix-arch" } diff --git a/README.md b/README.md index d71470cb4..9f8935b26 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,19 @@ To generate the API documentation locally, follow the instructions to generate d - [Go implementation of filecoin-proofs sectorbuilder API](https://github.com/filecoin-project/go-sectorbuilder/blob/master/sectorbuilder.go) and [associated interface structures](https://github.com/filecoin-project/go-sectorbuilder/blob/master/interface.go). +## Building for Arm64 + +In order to build for arm64 the current requirements are + +- nightly rust compiler + +Example for building `filecoin-proofs` + +``` +$ rustup +nightly target add aarch64-unknown-linux-gnu +$ cargo +nightly build -p filecoin-proofs --release --target aarch64-unknown-linux-gnu +``` + ## Contributing See [Contributing](CONTRIBUTING.md) diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index cccc3963c..b5e6e96cd 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -34,7 +34,6 @@ cpu-time = "1.0.0" git2 = "0.13.6" heim = { version = "0.1.0-beta.1", features = ["host", "memory", "cpu"] } async-std = "1.6" -raw-cpuid = "7.0.3" blake2s_simd = "0.5.6" fil_logger = "0.1" log = "0.4.8" @@ -55,3 +54,7 @@ default = ["gpu", "measurements"] gpu = ["storage-proofs/gpu", "filecoin-proofs/gpu", "bellperson/gpu", "fil-sapling-crypto/gpu"] measurements = ["storage-proofs/measurements"] profile = ["storage-proofs/profile", "measurements"] + + +[target.'cfg(target_arch = "x86_64")'.dependencies] +raw-cpuid = "7.0.3" diff --git a/fil-proofs-tooling/src/metadata.rs b/fil-proofs-tooling/src/metadata.rs index c653dd610..03cb6af23 100644 --- a/fil-proofs-tooling/src/metadata.rs +++ b/fil-proofs-tooling/src/metadata.rs @@ -77,20 +77,38 @@ impl SystemMetadata { let cpu_physical = block_on(async { heim::cpu::physical_count().await }) .map_err(|_| anyhow!("Failed to retrieve cpu physical count information"))?; - let cpuid = raw_cpuid::CpuId::new(); - let processor = cpuid - .get_extended_function_info() - .and_then(|info| info.processor_brand_string().map(|s| s.to_string())) - .unwrap_or_default(); - let (base, max) = cpuid - .get_processor_frequency_info() - .map(|info| { + let (processor, base, max, features) = { + #[cfg(target_arch = "x86_64")] + { + let cpuid = raw_cpuid::CpuId::new(); + let processor = cpuid + .get_extended_function_info() + .and_then(|info| info.processor_brand_string().map(|s| s.to_string())) + .unwrap_or_default(); + let (base, max) = cpuid + .get_processor_frequency_info() + .map(|info| { + ( + info.processor_base_frequency(), + info.processor_max_frequency(), + ) + }) + .unwrap_or_default(); ( - info.processor_base_frequency(), - info.processor_max_frequency(), + processor, + base, + max, + cpuid + .get_feature_info() + .map(|info| format!("{:?}", info)) + .unwrap_or_default(), ) - }) - .unwrap_or_default(); + } + #[cfg(not(target_arch = "x86_64"))] + { + ("unknown".into(), 0, 0, "unknown".into()) + } + }; Ok(SystemMetadata { system: host.system().into(), @@ -100,10 +118,7 @@ impl SystemMetadata { processor, processor_base_frequency_hz: base, processor_max_frequency_hz: max, - processor_features: cpuid - .get_feature_info() - .map(|info| format!("{:?}", info)) - .unwrap_or_default(), + processor_features: features, processor_cores_logical: cpu_logical, processor_cores_physical: cpu_physical.unwrap_or_default(), memory_total_bytes: memory.total().get::(), diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 3b1c5487a..0586f5e72 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -44,7 +44,7 @@ merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" rand_xorshift = "0.2.0" -sha2 = { version = "0.8.3", package = "sha2ni" } +sha2 = "0.9.1" typenum = "1.11.2" bitintr = "0.3.0" gperftools = { version = "0.2", optional = true } diff --git a/sha2raw/Cargo.toml b/sha2raw/Cargo.toml index e42305615..89707b0c9 100644 --- a/sha2raw/Cargo.toml +++ b/sha2raw/Cargo.toml @@ -11,24 +11,26 @@ categories = ["cryptography", "no-std"] edition = "2018" [dependencies] -digest = "0.8" +digest = "0.9" block-buffer = "0.7" fake-simd = "0.1" opaque-debug = "0.2" sha2-asm = { version = "0.5", optional = true } -raw-cpuid = "7.0.3" [dependencies.lazy_static] version = "1.4.0" +[target.'cfg(target_arch = "x86_64")'.dependencies] +cpuid-bool = "0.1.0" + [dev-dependencies] -digest = { version = "0.8", features = ["dev", "std"] } -sha2 = "0.8.1" +digest = { version = "0.9", features = ["dev", "std"] } +sha2 = "0.9.1" rand = "0.7.3" rand_xorshift = "0.2.0" [features] -default = [] +default = ["asm"] asm = ["sha2-asm"] diff --git a/sha2raw/src/platform.rs b/sha2raw/src/platform.rs index b17a86f35..81293f9b4 100644 --- a/sha2raw/src/platform.rs +++ b/sha2raw/src/platform.rs @@ -1,5 +1,3 @@ -use raw_cpuid::CpuId; - #[allow(dead_code)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum Platform { @@ -16,7 +14,7 @@ pub struct Implementation(Platform); impl Implementation { pub fn detect() -> Self { // Try the different implementations in order of how fast/modern they are. - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[cfg(target_arch = "x86_64")] { if let Some(sha_impl) = Self::sha_if_supported() { return sha_impl; @@ -36,16 +34,12 @@ impl Implementation { Implementation(Platform::Portable) } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[cfg(target_arch = "x86_64")] #[allow(unreachable_code)] pub fn sha_if_supported() -> Option { // Use raw_cpuid instead of is_x86_feature_detected, to ensure the check // never happens at compile time. - let cpuid = CpuId::new(); - let is_runtime_ok = cpuid - .get_extended_feature_info() - .map(|info| info.has_sha()) - .unwrap_or_default(); + let is_runtime_ok = cpuid_bool::cpuid_bool!("sha"); #[cfg(target_feature = "sha")] { diff --git a/storage-proofs/core/Cargo.toml b/storage-proofs/core/Cargo.toml index a6342dbe3..f64f5381b 100644 --- a/storage-proofs/core/Cargo.toml +++ b/storage-proofs/core/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.2" memmap = "0.7" aes = "0.3" block-modes = "0.3" -sha2 = { version = "0.8.3", package = "sha2ni" } +sha2 = "0.9.1" tempfile = "3" fs2 = "0.4" rayon = "1.0.0" diff --git a/storage-proofs/core/src/crypto/mod.rs b/storage-proofs/core/src/crypto/mod.rs index 87cd38f33..37380fed3 100644 --- a/storage-proofs/core/src/crypto/mod.rs +++ b/storage-proofs/core/src/crypto/mod.rs @@ -17,6 +17,6 @@ pub fn derive_porep_domain_seed( Sha256::new() .chain(domain_separation_tag.0) .chain(porep_id) - .result() + .finalize() .into() } diff --git a/storage-proofs/core/src/crypto/pedersen.rs b/storage-proofs/core/src/crypto/pedersen.rs index 830237915..dc707f847 100644 --- a/storage-proofs/core/src/crypto/pedersen.rs +++ b/storage-proofs/core/src/crypto/pedersen.rs @@ -1,12 +1,12 @@ use anyhow::{ensure, Context}; use ff::PrimeFieldRepr; use fil_sapling_crypto::jubjub::JubjubBls12; -use fil_sapling_crypto::pedersen_hash::Personalization; use lazy_static::lazy_static; -use paired::bls12_381::{Bls12, Fr, FrRepr}; +use paired::bls12_381::{Fr, FrRepr}; use crate::error::Result; use crate::fr32::bytes_into_frs; +use crate::hasher::pedersen::pedersen_hash; use crate::settings; lazy_static! { @@ -26,14 +26,7 @@ pub fn pedersen(data: &[u8]) -> Fr { } pub fn pedersen_bits<'a, S: Iterator>(data: Bits<&'a [u8], S>) -> Fr { - let digest = if cfg!(target_arch = "x86_64") { - use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp; - pedersen_hash_bls12_381_with_precomp::<_>(Personalization::None, data, &JJ_PARAMS) - } else { - use fil_sapling_crypto::pedersen_hash::pedersen_hash; - pedersen_hash::(Personalization::None, data, &JJ_PARAMS) - }; - + let digest = pedersen_hash(data); digest.into_xy().0 } @@ -70,14 +63,7 @@ fn pedersen_compression_bits(bits: T) -> FrRepr where T: IntoIterator, { - let digest = if cfg!(target_arch = "x86_64") { - use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp; - pedersen_hash_bls12_381_with_precomp::<_>(Personalization::None, bits, &JJ_PARAMS) - } else { - use fil_sapling_crypto::pedersen_hash::pedersen_hash; - pedersen_hash::(Personalization::None, bits, &JJ_PARAMS) - }; - + let digest = pedersen_hash(bits); digest.into_xy().0.into() } diff --git a/storage-proofs/core/src/drgraph.rs b/storage-proofs/core/src/drgraph.rs index 1b309676d..478218ab8 100644 --- a/storage-proofs/core/src/drgraph.rs +++ b/storage-proofs/core/src/drgraph.rs @@ -112,17 +112,17 @@ impl Graph for BucketGraph { _exp_parents_data: Option<&[u8]>, ) -> Result { let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(id)); + hasher.update(AsRef::<[u8]>::as_ref(id)); // The hash is about the parents, hence skip if a node doesn't have any parents if node != parents[0] as usize { for parent in parents.iter() { let offset = data_at_node_offset(*parent as usize); - hasher.input(&base_parents_data[offset..offset + NODE_SIZE]); + hasher.update(&base_parents_data[offset..offset + NODE_SIZE]); } } - let hash = hasher.result(); + let hash = hasher.finalize(); Ok(bytes_into_fr_repr_safe(hash.as_ref()).into()) } diff --git a/storage-proofs/core/src/hasher/pedersen.rs b/storage-proofs/core/src/hasher/pedersen.rs index d4b6e107e..564ea5149 100644 --- a/storage-proofs/core/src/hasher/pedersen.rs +++ b/storage-proofs/core/src/hasher/pedersen.rs @@ -190,6 +190,25 @@ impl StdHasher for PedersenFunction { } } +pub fn pedersen_hash>( + node_bits: I, +) -> fil_sapling_crypto::jubjub::edwards::Point { + #[cfg(target_arch = "x86_64")] + { + use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp; + pedersen_hash_bls12_381_with_precomp::<_>( + Personalization::None, + node_bits, + &pedersen::JJ_PARAMS, + ) + } + #[cfg(not(target_arch = "x86_64"))] + { + use fil_sapling_crypto::pedersen_hash::pedersen_hash; + pedersen_hash::(Personalization::None, node_bits, &pedersen::JJ_PARAMS) + } +} + impl HashFunction for PedersenFunction { fn hash(data: &[u8]) -> PedersenDomain { pedersen::pedersen_md_no_padding(data).into() @@ -197,18 +216,7 @@ impl HashFunction for PedersenFunction { fn hash2(a: &PedersenDomain, b: &PedersenDomain) -> PedersenDomain { let data = NodeBits::new(&(a.0).0[..], &(b.0).0[..]); - - let digest = if cfg!(target_arch = "x86_64") { - use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp; - pedersen_hash_bls12_381_with_precomp::<_>( - Personalization::None, - data, - &pedersen::JJ_PARAMS, - ) - } else { - use fil_sapling_crypto::pedersen_hash::pedersen_hash; - pedersen_hash::(Personalization::None, data, &pedersen::JJ_PARAMS) - }; + let digest = pedersen_hash(data); digest.into_xy().0.into() } @@ -319,18 +327,7 @@ impl LightAlgorithm for PedersenFunction { _height: usize, ) -> PedersenDomain { let node_bits = NodeBits::new(&(left.0).0[..], &(right.0).0[..]); - - let digest = if cfg!(target_arch = "x86_64") { - use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp; - pedersen_hash_bls12_381_with_precomp::<_>( - Personalization::None, - node_bits, - &pedersen::JJ_PARAMS, - ) - } else { - use fil_sapling_crypto::pedersen_hash::pedersen_hash; - pedersen_hash::(Personalization::None, node_bits, &pedersen::JJ_PARAMS) - }; + let digest = pedersen_hash(node_bits); digest.into_xy().0.into() } diff --git a/storage-proofs/core/src/hasher/sha256.rs b/storage-proofs/core/src/hasher/sha256.rs index 3e6d02a6b..0a00d2769 100644 --- a/storage-proofs/core/src/hasher/sha256.rs +++ b/storage-proofs/core/src/hasher/sha256.rs @@ -47,7 +47,7 @@ pub struct Sha256Function(Sha256); impl StdHasher for Sha256Function { #[inline] fn write(&mut self, msg: &[u8]) { - self.0.input(msg) + self.0.update(msg) } #[inline] @@ -179,7 +179,7 @@ impl HashFunction for Sha256Function { let hashed = Sha256::new() .chain(AsRef::<[u8]>::as_ref(a)) .chain(AsRef::<[u8]>::as_ref(b)) - .result(); + .finalize(); let mut res = Sha256Domain::default(); res.0.copy_from_slice(&hashed[..]); res.trim_to_fr32(); @@ -303,7 +303,7 @@ impl Algorithm for Sha256Function { #[inline] fn hash(&mut self) -> Sha256Domain { let mut h = [0u8; 32]; - h.copy_from_slice(self.0.clone().result().as_ref()); + h.copy_from_slice(self.0.clone().finalize().as_ref()); let mut dd = Sha256Domain::from(h); dd.trim_to_fr32(); dd diff --git a/storage-proofs/core/src/parameter_cache.rs b/storage-proofs/core/src/parameter_cache.rs index 881754373..d092ef7a1 100644 --- a/storage-proofs/core/src/parameter_cache.rs +++ b/storage-proofs/core/src/parameter_cache.rs @@ -170,8 +170,8 @@ where let param_identifier = pub_params.identifier(); info!("parameter set identifier for cache: {}", param_identifier); let mut hasher = Sha256::default(); - hasher.input(¶m_identifier.into_bytes()); - let circuit_hash = hasher.result(); + hasher.update(¶m_identifier.into_bytes()); + let circuit_hash = hasher.finalize(); format!( "{}-{:02x}", Self::cache_prefix(), diff --git a/storage-proofs/porep/Cargo.toml b/storage-proofs/porep/Cargo.toml index 38bc3f1ed..1fa9ea73b 100644 --- a/storage-proofs/porep/Cargo.toml +++ b/storage-proofs/porep/Cargo.toml @@ -16,7 +16,7 @@ merkletree = "0.21.0" memmap = "0.7" num-bigint = "0.2" num-traits = "0.2" -sha2 = { version = "0.8.3", package = "sha2ni" } +sha2 = "0.9.1" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} ff = { version = "0.2.1", package = "fff" } diff --git a/storage-proofs/porep/src/drg/vanilla.rs b/storage-proofs/porep/src/drg/vanilla.rs index aca96ef38..ef453ef88 100644 --- a/storage-proofs/porep/src/drg/vanilla.rs +++ b/storage-proofs/porep/src/drg/vanilla.rs @@ -396,13 +396,13 @@ where let key = { let prover_bytes = pub_inputs.replica_id.context("missing replica_id")?; - hasher.input(AsRef::<[u8]>::as_ref(&prover_bytes)); + hasher.update(AsRef::<[u8]>::as_ref(&prover_bytes)); for p in proof.replica_parents[i].iter() { - hasher.input(AsRef::<[u8]>::as_ref(&p.1.data)); + hasher.update(AsRef::<[u8]>::as_ref(&p.1.data)); } - let hash = hasher.result_reset(); + let hash = hasher.finalize_reset(); bytes_into_fr_repr_safe(hash.as_ref()).into() }; @@ -580,18 +580,18 @@ pub fn create_key_from_tree( tree: &LCMerkleTree, ) -> Result { let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(&id)); + hasher.update(AsRef::<[u8]>::as_ref(&id)); // The hash is about the parents, hence skip if a node doesn't have any parents if node != parents[0] as usize { let mut scratch: [u8; NODE_SIZE] = [0; NODE_SIZE]; for parent in parents.iter() { tree.read_into(*parent as usize, &mut scratch)?; - hasher.input(&scratch); + hasher.update(&scratch); } } - let hash = hasher.result(); + let hash = hasher.finalize(); Ok(bytes_into_fr_repr_safe(hash.as_ref()).into()) } diff --git a/storage-proofs/porep/src/stacked/vanilla/cache.rs b/storage-proofs/porep/src/stacked/vanilla/cache.rs index 757813569..fe1b1d39a 100644 --- a/storage-proofs/porep/src/stacked/vanilla/cache.rs +++ b/storage-proofs/porep/src/stacked/vanilla/cache.rs @@ -247,13 +247,13 @@ where { let mut hasher = Sha256::default(); - hasher.input(H::name()); - hasher.input(graph.identifier()); + hasher.update(H::name()); + hasher.update(graph.identifier()); for key in &graph.feistel_keys { - hasher.input(key.to_le_bytes()); + hasher.update(key.to_le_bytes()); } - hasher.input(cache_entries.to_le_bytes()); - let h = hasher.result(); + hasher.update(cache_entries.to_le_bytes()); + let h = hasher.finalize(); PathBuf::from(parent_cache_dir_name()).join(format!( "v{}-sdr-parent-{}.cache", VERSION, diff --git a/storage-proofs/porep/src/stacked/vanilla/challenges.rs b/storage-proofs/porep/src/stacked/vanilla/challenges.rs index 78923bafc..d35026ca3 100644 --- a/storage-proofs/porep/src/stacked/vanilla/challenges.rs +++ b/storage-proofs/porep/src/stacked/vanilla/challenges.rs @@ -55,7 +55,7 @@ impl LayerChallenges { .chain(replica_id.into_bytes()) .chain(seed) .chain(&j.to_le_bytes()) - .result(); + .finalize(); let big_challenge = BigUint::from_bytes_le(hash.as_ref()); diff --git a/storage-proofs/porep/src/stacked/vanilla/create_label.rs b/storage-proofs/porep/src/stacked/vanilla/create_label.rs index 1299225df..634a62f2a 100644 --- a/storage-proofs/porep/src/stacked/vanilla/create_label.rs +++ b/storage-proofs/porep/src/stacked/vanilla/create_label.rs @@ -1,8 +1,3 @@ -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - use sha2raw::Sha256; use storage_proofs_core::{ error::Result, @@ -31,9 +26,7 @@ pub fn create_label( let hash = if node > 0 { // prefetch previous node, which is always a parent let prev = &layer_labels[(node - 1) * NODE_SIZE..node * NODE_SIZE]; - unsafe { - _mm_prefetch(prev.as_ptr() as *const i8, _MM_HINT_T0); - } + prefetch!(prev.as_ptr() as *const i8); graph.copy_parents_data(node as u32, &*layer_labels, hasher, cache)? } else { @@ -71,9 +64,7 @@ pub fn create_label_exp( let hash = if node > 0 { // prefetch previous node, which is always a parent let prev = &layer_labels[(node - 1) * NODE_SIZE..node * NODE_SIZE]; - unsafe { - _mm_prefetch(prev.as_ptr() as *const i8, _MM_HINT_T0); - } + prefetch!(prev.as_ptr() as *const i8); graph.copy_parents_data_exp(node as u32, &*layer_labels, exp_parents_data, hasher, cache)? } else { diff --git a/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs b/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs index 6863c4b66..fe48af650 100644 --- a/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/encoding_proof.rs @@ -39,14 +39,14 @@ impl EncodingProof { // node id buffer[36..44].copy_from_slice(&(self.node as u64).to_be_bytes()); - hasher.input(&buffer[..]); + hasher.update(&buffer[..]); // parents for parent in &self.parents { - hasher.input(AsRef::<[u8]>::as_ref(parent)); + hasher.update(AsRef::<[u8]>::as_ref(parent)); } - bytes_into_fr_repr_safe(hasher.result().as_ref()).into() + bytes_into_fr_repr_safe(hasher.finalize().as_ref()).into() } pub fn verify( diff --git a/storage-proofs/porep/src/stacked/vanilla/graph.rs b/storage-proofs/porep/src/stacked/vanilla/graph.rs index 4b76ba63a..7bd6be58a 100644 --- a/storage-proofs/porep/src/stacked/vanilla/graph.rs +++ b/storage-proofs/porep/src/stacked/vanilla/graph.rs @@ -1,11 +1,6 @@ use std::convert::TryInto; use std::marker::PhantomData; -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - use anyhow::ensure; use log::info; use sha2raw::Sha256; @@ -68,9 +63,7 @@ fn prefetch(parents: &[u32], data: &[u8]) { let start = *parent as usize * NODE_SIZE; let end = start + NODE_SIZE; - unsafe { - _mm_prefetch(data[start..end].as_ptr() as *const i8, _MM_HINT_T0); - } + prefetch!(data[start..end].as_ptr() as *const i8); } } diff --git a/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs b/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs index 98ef3d594..a7038fc73 100644 --- a/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/labeling_proof.rs @@ -37,15 +37,15 @@ impl LabelingProof { // node id buffer[36..44].copy_from_slice(&(self.node as u64).to_be_bytes()); - hasher.input(&buffer[..]); + hasher.update(&buffer[..]); // parents for parent in &self.parents { let data = AsRef::<[u8]>::as_ref(parent); - hasher.input(data); + hasher.update(data); } - bytes_into_fr_repr_safe(hasher.result().as_ref()).into() + bytes_into_fr_repr_safe(hasher.finalize().as_ref()).into() } pub fn verify(&self, replica_id: &H::Domain, expected_label: &H::Domain) -> bool { diff --git a/storage-proofs/porep/src/stacked/vanilla/macros.rs b/storage-proofs/porep/src/stacked/vanilla/macros.rs index 020d9a4b8..bdb5a5ae8 100644 --- a/storage-proofs/porep/src/stacked/vanilla/macros.rs +++ b/storage-proofs/porep/src/stacked/vanilla/macros.rs @@ -43,3 +43,17 @@ macro_rules! check { } }; } + +macro_rules! prefetch { + ($val:expr) => { + #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] + unsafe { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + _mm_prefetch($val, _MM_HINT_T0); + } + }; +} diff --git a/storage-proofs/porep/src/stacked/vanilla/params.rs b/storage-proofs/porep/src/stacked/vanilla/params.rs index a575dcbc4..c04bc4819 100644 --- a/storage-proofs/porep/src/stacked/vanilla/params.rs +++ b/storage-proofs/porep/src/stacked/vanilla/params.rs @@ -747,7 +747,7 @@ pub fn generate_replica_id>( .chain(ticket) .chain(AsRef::<[u8]>::as_ref(&comm_d)) .chain(porep_seed) - .result(); + .finalize(); bytes_into_fr_repr_safe(hash.as_ref()).into() } diff --git a/storage-proofs/post/Cargo.toml b/storage-proofs/post/Cargo.toml index 994de2495..eb4497a14 100644 --- a/storage-proofs/post/Cargo.toml +++ b/storage-proofs/post/Cargo.toml @@ -14,7 +14,7 @@ rand = "0.7" merkletree = "0.21.0" byteorder = "1" crossbeam = "0.7.3" -sha2 = { version = "0.8.3", package = "sha2ni" } +sha2 = "0.9.1" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} blake2b_simd = "0.5" diff --git a/storage-proofs/post/src/election/vanilla.rs b/storage-proofs/post/src/election/vanilla.rs index f5ae4d974..1a17dd87c 100644 --- a/storage-proofs/post/src/election/vanilla.rs +++ b/storage-proofs/post/src/election/vanilla.rs @@ -267,11 +267,11 @@ pub fn generate_sector_challenge( sectors: &OrderedSectorSet, ) -> Result { let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(&randomness)); - hasher.input(&n.to_le_bytes()[..]); - let hash = hasher.result(); + hasher.update(AsRef::<[u8]>::as_ref(&randomness)); + hasher.update(&n.to_le_bytes()[..]); + let hash = hasher.finalize(); - let sector_challenge = LittleEndian::read_u64(&hash.as_ref()[..8]); + let sector_challenge = LittleEndian::read_u64(&hash[..8]); let sector_index = (sector_challenge % sectors.len() as u64) as usize; let sector = *sectors .iter() @@ -317,12 +317,12 @@ pub fn generate_leaf_challenge( ); let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(&randomness)); - hasher.input(§or_challenge_index.to_le_bytes()[..]); - hasher.input(&leaf_challenge_index.to_le_bytes()[..]); - let hash = hasher.result(); + hasher.update(AsRef::<[u8]>::as_ref(&randomness)); + hasher.update(§or_challenge_index.to_le_bytes()[..]); + hasher.update(&leaf_challenge_index.to_le_bytes()[..]); + let hash = hasher.finalize(); - let leaf_challenge = LittleEndian::read_u64(&hash.as_ref()[..8]); + let leaf_challenge = LittleEndian::read_u64(&hash[..8]); let challenged_range_index = leaf_challenge % (pub_params.sector_size / (pub_params.challenged_nodes * NODE_SIZE) as u64); diff --git a/storage-proofs/post/src/fallback/vanilla.rs b/storage-proofs/post/src/fallback/vanilla.rs index 0e2264360..5a6ad3ec0 100644 --- a/storage-proofs/post/src/fallback/vanilla.rs +++ b/storage-proofs/post/src/fallback/vanilla.rs @@ -176,13 +176,13 @@ pub fn generate_sector_challenge( prover_id: T, ) -> Result { let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(&prover_id)); - hasher.input(AsRef::<[u8]>::as_ref(&randomness)); - hasher.input(&n.to_le_bytes()[..]); + hasher.update(AsRef::<[u8]>::as_ref(&prover_id)); + hasher.update(AsRef::<[u8]>::as_ref(&randomness)); + hasher.update(&n.to_le_bytes()[..]); - let hash = hasher.result(); + let hash = hasher.finalize(); - let sector_challenge = LittleEndian::read_u64(&hash.as_ref()[..8]); + let sector_challenge = LittleEndian::read_u64(&hash[..8]); let sector_index = sector_challenge % sector_set_len; Ok(sector_index) @@ -218,12 +218,12 @@ pub fn generate_leaf_challenge( leaf_challenge_index: u64, ) -> Result { let mut hasher = Sha256::new(); - hasher.input(AsRef::<[u8]>::as_ref(&randomness)); - hasher.input(§or_id.to_le_bytes()[..]); - hasher.input(&leaf_challenge_index.to_le_bytes()[..]); - let hash = hasher.result(); + hasher.update(AsRef::<[u8]>::as_ref(&randomness)); + hasher.update(§or_id.to_le_bytes()[..]); + hasher.update(&leaf_challenge_index.to_le_bytes()[..]); + let hash = hasher.finalize(); - let leaf_challenge = LittleEndian::read_u64(&hash.as_ref()[..8]); + let leaf_challenge = LittleEndian::read_u64(&hash[..8]); let challenged_range_index = leaf_challenge % (pub_params.sector_size / NODE_SIZE as u64); From 4d2a936b56c5ceeb39d389d3bd9d2cba5e20c40b Mon Sep 17 00:00:00 2001 From: Keyvan Kambakhsh Date: Fri, 24 Jul 2020 00:04:25 +0430 Subject: [PATCH 2/2] upload deps --- Cargo.toml | 4 ---- fil-proofs-tooling/Cargo.toml | 4 ++-- filecoin-proofs/Cargo.toml | 4 ++-- storage-proofs/core/Cargo.toml | 4 ++-- storage-proofs/porep/Cargo.toml | 5 ++--- storage-proofs/post/Cargo.toml | 4 ++-- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 94beff80d..0a0b6acae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,3 @@ members = [ "fil-proofs-tooling", "sha2raw" ] - -[patch.crates-io] -fil-sapling-crypto = { git = "https://github.com/filecoin-project/sapling-crypto", branch = "fix/compile-nonx86" } -fff = { git = "https://github.com/filecoin-project/ff", branch = "fix-arch" } diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index b5e6e96cd..b3682fba4 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -25,7 +25,7 @@ chrono = { version = "0.4.7", features = ["serde"] } memmap = "0.7.0" bellperson = "0.9.1" paired = "0.20.0" -fil-sapling-crypto = "0.6.0" +fil-sapling-crypto = "0.6.3" rand = "0.7" storage-proofs = { path = "../storage-proofs"} filecoin-proofs = { path = "../filecoin-proofs"} @@ -41,7 +41,7 @@ uom = "0.28" merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" -ff = { version = "0.2.0", package = "fff" } +ff = { version = "0.2.3", package = "fff" } rand_xorshift = "0.2.0" bytefmt = "0.1.7" rayon = "1.3.0" diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 0586f5e72..65e003276 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -23,11 +23,11 @@ serde_cbor = "0.11.1" serde = { version = "1.0", features = ["rc", "derive"] } serde_json = "1.0" regex = "1.3.7" -ff = { version = "0.2.1", package = "fff" } +ff = { version = "0.2.3", package = "fff" } blake2b_simd = "0.5" bellperson = "0.9.1" paired = "0.20.0" -fil-sapling-crypto = "0.6.0" +fil-sapling-crypto = "0.6.3" clap = "2" log = "0.4.7" fil_logger = "0.1" diff --git a/storage-proofs/core/Cargo.toml b/storage-proofs/core/Cargo.toml index f64f5381b..3700e0e7d 100644 --- a/storage-proofs/core/Cargo.toml +++ b/storage-proofs/core/Cargo.toml @@ -29,10 +29,10 @@ serde = { version = "1.0", features = ["derive"]} blake2b_simd = "0.5" blake2s_simd = "0.5" toml = "0.5" -ff = { version = "0.2.1", package = "fff" } +ff = { version = "0.2.3", package = "fff" } bellperson = "0.9.1" paired = { version = "0.20.0", features = ["serde"] } -fil-sapling-crypto = "0.6.0" +fil-sapling-crypto = "0.6.3" serde_json = "1.0" log = "0.4.7" rand_chacha = "0.2.1" diff --git a/storage-proofs/porep/Cargo.toml b/storage-proofs/porep/Cargo.toml index 1fa9ea73b..a62beef86 100644 --- a/storage-proofs/porep/Cargo.toml +++ b/storage-proofs/porep/Cargo.toml @@ -19,10 +19,10 @@ num-traits = "0.2" sha2 = "0.9.1" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} -ff = { version = "0.2.1", package = "fff" } +ff = { version = "0.2.3", package = "fff" } bellperson = "0.9.1" paired = { version = "0.20.0", features = ["serde"] } -fil-sapling-crypto = "0.6.0" +fil-sapling-crypto = "0.6.3" log = "0.4.7" pretty_assertions = "0.6.1" generic-array = "0.13.2" @@ -49,4 +49,3 @@ harness = false [[bench]] name = "parents" harness = false - diff --git a/storage-proofs/post/Cargo.toml b/storage-proofs/post/Cargo.toml index eb4497a14..7eca9ec62 100644 --- a/storage-proofs/post/Cargo.toml +++ b/storage-proofs/post/Cargo.toml @@ -19,10 +19,10 @@ rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} blake2b_simd = "0.5" blake2s_simd = "0.5" -ff = { version = "0.2.1", package = "fff" } +ff = { version = "0.2.3", package = "fff" } bellperson = "0.9.1" paired = { version = "0.20.0", features = ["serde"] } -fil-sapling-crypto = "0.6.0" +fil-sapling-crypto = "0.6.3" log = "0.4.7" hex = "0.4.0" generic-array = "0.13.2"