feat: allow for compilation on aarch64 (#1204)

* feat: allow for compilation on aarch64 sha2raw: update deps and enable asm by default fix compile time imports move prefetch into macro fixup fil-proofs-tooling Co-authored-by: Keyvan Kambakhsh <keyvankambakhsh@gmail.com>
filecoin-project · Jul 23, 2020 · 8910194 · 8910194
1 parent 8d94c59
commit 8910194
Show file tree

Hide file tree

Showing 26 changed files with 161 additions and 154 deletions.
diff --git a/README.md b/README.md
@@ -258,6 +258,19 @@ To generate the API documentation locally, follow the instructions to generate d
 - [Go implementation of filecoin-proofs sectorbuilder API](https://github.com/filecoin-project/go-sectorbuilder/blob/master/sectorbuilder.go) and [associated interface structures](https://github.com/filecoin-project/go-sectorbuilder/blob/master/interface.go).
 
 
+## Building for Arm64
+
+In order to build for arm64 the current requirements are
+
+- nightly rust compiler
+
+Example for building `filecoin-proofs`
+
+```
+$ rustup +nightly target add aarch64-unknown-linux-gnu
+$ cargo +nightly build -p filecoin-proofs --release --target aarch64-unknown-linux-gnu
+```
+
 ## Contributing
 
 See [Contributing](CONTRIBUTING.md)

diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml
@@ -25,7 +25,7 @@ chrono = { version = "0.4.7", features = ["serde"] }
 memmap = "0.7.0"
 bellperson = "0.9.1"
 paired = "0.20.0"
-fil-sapling-crypto = "0.6.0"
+fil-sapling-crypto = "0.6.3"
 rand = "0.7"
 storage-proofs = { path = "../storage-proofs"}
 filecoin-proofs = { path = "../filecoin-proofs"}
@@ -34,15 +34,14 @@ cpu-time = "1.0.0"
 git2 = "0.13.6"
 heim = { version = "0.1.0-beta.1", features = ["host", "memory", "cpu"] }
 async-std = "1.6"
-raw-cpuid = "7.0.3"
 blake2s_simd = "0.5.6"
 fil_logger = "0.1"
 log = "0.4.8"
 uom = "0.28"
 merkletree = "0.21.0"
 bincode = "1.1.2"
 anyhow = "1.0.23"
-ff = { version = "0.2.0", package = "fff" }
+ff = { version = "0.2.3", package = "fff" }
 rand_xorshift = "0.2.0"
 bytefmt = "0.1.7"
 rayon = "1.3.0"
@@ -55,3 +54,7 @@ default = ["gpu", "measurements"]
 gpu = ["storage-proofs/gpu", "filecoin-proofs/gpu", "bellperson/gpu", "fil-sapling-crypto/gpu"]
 measurements = ["storage-proofs/measurements"]
 profile = ["storage-proofs/profile", "measurements"]
+
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+raw-cpuid = "7.0.3"
diff --git a/fil-proofs-tooling/src/metadata.rs b/fil-proofs-tooling/src/metadata.rs
@@ -77,20 +77,38 @@ impl SystemMetadata {
         let cpu_physical = block_on(async { heim::cpu::physical_count().await })
             .map_err(|_| anyhow!("Failed to retrieve cpu physical count information"))?;
 
-        let cpuid = raw_cpuid::CpuId::new();
-        let processor = cpuid
-            .get_extended_function_info()
-            .and_then(|info| info.processor_brand_string().map(|s| s.to_string()))
-            .unwrap_or_default();
-        let (base, max) = cpuid
-            .get_processor_frequency_info()
-            .map(|info| {
+        let (processor, base, max, features) = {
+            #[cfg(target_arch = "x86_64")]
+            {
+                let cpuid = raw_cpuid::CpuId::new();
+                let processor = cpuid
+                    .get_extended_function_info()
+                    .and_then(|info| info.processor_brand_string().map(|s| s.to_string()))
+                    .unwrap_or_default();
+                let (base, max) = cpuid
+                    .get_processor_frequency_info()
+                    .map(|info| {
+                        (
+                            info.processor_base_frequency(),
+                            info.processor_max_frequency(),
+                        )
+                    })
+                    .unwrap_or_default();
                 (
-                    info.processor_base_frequency(),
-                    info.processor_max_frequency(),
+                    processor,
+                    base,
+                    max,
+                    cpuid
+                        .get_feature_info()
+                        .map(|info| format!("{:?}", info))
+                        .unwrap_or_default(),
                 )
-            })
-            .unwrap_or_default();
+            }
+            #[cfg(not(target_arch = "x86_64"))]
+            {
+                ("unknown".into(), 0, 0, "unknown".into())
+            }
+        };
 
         Ok(SystemMetadata {
             system: host.system().into(),
@@ -100,10 +118,7 @@ impl SystemMetadata {
             processor,
             processor_base_frequency_hz: base,
             processor_max_frequency_hz: max,
-            processor_features: cpuid
-                .get_feature_info()
-                .map(|info| format!("{:?}", info))
-                .unwrap_or_default(),
+            processor_features: features,
             processor_cores_logical: cpu_logical,
             processor_cores_physical: cpu_physical.unwrap_or_default(),
             memory_total_bytes: memory.total().get::<uom::si::information::byte>(),

diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml
@@ -23,11 +23,11 @@ serde_cbor = "0.11.1"
 serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 regex = "1.3.7"
-ff = { version = "0.2.1", package = "fff" }
+ff = { version = "0.2.3", package = "fff" }
 blake2b_simd = "0.5"
 bellperson = "0.9.1"
 paired = "0.20.0"
-fil-sapling-crypto = "0.6.0"
+fil-sapling-crypto = "0.6.3"
 clap = "2"
 log = "0.4.7"
 fil_logger = "0.1"
@@ -44,7 +44,7 @@ merkletree = "0.21.0"
 bincode = "1.1.2"
 anyhow = "1.0.23"
 rand_xorshift = "0.2.0"
-sha2 = { version = "0.8.3", package = "sha2ni" }
+sha2 = "0.9.1"
 typenum = "1.11.2"
 bitintr = "0.3.0"
 gperftools = { version = "0.2", optional = true }

diff --git a/sha2raw/Cargo.toml b/sha2raw/Cargo.toml
@@ -11,24 +11,26 @@ categories = ["cryptography", "no-std"]
 edition = "2018"
 
 [dependencies]
-digest = "0.8"
+digest = "0.9"
 block-buffer = "0.7"
 fake-simd = "0.1"
 opaque-debug = "0.2"
 sha2-asm = { version = "0.5", optional = true }
-raw-cpuid = "7.0.3"
 
 [dependencies.lazy_static]
 version = "1.4.0"
 
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+cpuid-bool = "0.1.0"
+
 [dev-dependencies]
-digest = { version = "0.8", features = ["dev", "std"] }
-sha2 = "0.8.1"
+digest = { version = "0.9", features = ["dev", "std"] }
+sha2 = "0.9.1"
 rand = "0.7.3"
 rand_xorshift = "0.2.0"
 
 [features]
-default = []
+default = ["asm"]
 asm = ["sha2-asm"]
 
 
diff --git a/sha2raw/src/platform.rs b/sha2raw/src/platform.rs
@@ -1,5 +1,3 @@
-use raw_cpuid::CpuId;
-
 #[allow(dead_code)]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 enum Platform {
@@ -16,7 +14,7 @@ pub struct Implementation(Platform);
 impl Implementation {
     pub fn detect() -> Self {
         // Try the different implementations in order of how fast/modern they are.
-        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        #[cfg(target_arch = "x86_64")]
         {
             if let Some(sha_impl) = Self::sha_if_supported() {
                 return sha_impl;
@@ -36,16 +34,12 @@ impl Implementation {
         Implementation(Platform::Portable)
     }
 
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    #[cfg(target_arch = "x86_64")]
     #[allow(unreachable_code)]
     pub fn sha_if_supported() -> Option<Self> {
         // Use raw_cpuid instead of is_x86_feature_detected, to ensure the check
         // never happens at compile time.
-        let cpuid = CpuId::new();
-        let is_runtime_ok = cpuid
-            .get_extended_feature_info()
-            .map(|info| info.has_sha())
-            .unwrap_or_default();
+        let is_runtime_ok = cpuid_bool::cpuid_bool!("sha");
 
         #[cfg(target_feature = "sha")]
         {

diff --git a/storage-proofs/core/Cargo.toml b/storage-proofs/core/Cargo.toml
@@ -21,18 +21,18 @@ lazy_static = "1.2"
 memmap = "0.7"
 aes = "0.3"
 block-modes = "0.3"
-sha2 = { version = "0.8.3", package = "sha2ni" }
+sha2 = "0.9.1"
 tempfile = "3"
 fs2 = "0.4"
 rayon = "1.0.0"
 serde = { version = "1.0", features = ["derive"]}
 blake2b_simd = "0.5"
 blake2s_simd = "0.5"
 toml = "0.5"
-ff = { version = "0.2.1", package = "fff" }
+ff = { version = "0.2.3", package = "fff" }
 bellperson = "0.9.1"
 paired = { version = "0.20.0", features = ["serde"] }
-fil-sapling-crypto = "0.6.0"
+fil-sapling-crypto = "0.6.3"
 serde_json = "1.0"
 log = "0.4.7"
 rand_chacha = "0.2.1"

diff --git a/storage-proofs/core/src/crypto/mod.rs b/storage-proofs/core/src/crypto/mod.rs
@@ -17,6 +17,6 @@ pub fn derive_porep_domain_seed(
     Sha256::new()
         .chain(domain_separation_tag.0)
         .chain(porep_id)
-        .result()
+        .finalize()
         .into()
 }
diff --git a/storage-proofs/core/src/crypto/pedersen.rs b/storage-proofs/core/src/crypto/pedersen.rs
@@ -1,12 +1,12 @@
 use anyhow::{ensure, Context};
 use ff::PrimeFieldRepr;
 use fil_sapling_crypto::jubjub::JubjubBls12;
-use fil_sapling_crypto::pedersen_hash::Personalization;
 use lazy_static::lazy_static;
-use paired::bls12_381::{Bls12, Fr, FrRepr};
+use paired::bls12_381::{Fr, FrRepr};
 
 use crate::error::Result;
 use crate::fr32::bytes_into_frs;
+use crate::hasher::pedersen::pedersen_hash;
 use crate::settings;
 
 lazy_static! {
@@ -26,14 +26,7 @@ pub fn pedersen(data: &[u8]) -> Fr {
 }
 
 pub fn pedersen_bits<'a, S: Iterator<Item = &'a [u8]>>(data: Bits<&'a [u8], S>) -> Fr {
-    let digest = if cfg!(target_arch = "x86_64") {
-        use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp;
-        pedersen_hash_bls12_381_with_precomp::<_>(Personalization::None, data, &JJ_PARAMS)
-    } else {
-        use fil_sapling_crypto::pedersen_hash::pedersen_hash;
-        pedersen_hash::<Bls12, _>(Personalization::None, data, &JJ_PARAMS)
-    };
-
+    let digest = pedersen_hash(data);
     digest.into_xy().0
 }
 
@@ -70,14 +63,7 @@ fn pedersen_compression_bits<T>(bits: T) -> FrRepr
 where
     T: IntoIterator<Item = bool>,
 {
-    let digest = if cfg!(target_arch = "x86_64") {
-        use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp;
-        pedersen_hash_bls12_381_with_precomp::<_>(Personalization::None, bits, &JJ_PARAMS)
-    } else {
-        use fil_sapling_crypto::pedersen_hash::pedersen_hash;
-        pedersen_hash::<Bls12, _>(Personalization::None, bits, &JJ_PARAMS)
-    };
-
+    let digest = pedersen_hash(bits);
     digest.into_xy().0.into()
 }
 

diff --git a/storage-proofs/core/src/drgraph.rs b/storage-proofs/core/src/drgraph.rs
@@ -112,17 +112,17 @@ impl<H: Hasher> Graph<H> for BucketGraph<H> {
         _exp_parents_data: Option<&[u8]>,
     ) -> Result<Self::Key> {
         let mut hasher = Sha256::new();
-        hasher.input(AsRef::<[u8]>::as_ref(id));
+        hasher.update(AsRef::<[u8]>::as_ref(id));
 
         // The hash is about the parents, hence skip if a node doesn't have any parents
         if node != parents[0] as usize {
             for parent in parents.iter() {
                 let offset = data_at_node_offset(*parent as usize);
-                hasher.input(&base_parents_data[offset..offset + NODE_SIZE]);
+                hasher.update(&base_parents_data[offset..offset + NODE_SIZE]);
             }
         }
 
-        let hash = hasher.result();
+        let hash = hasher.finalize();
         Ok(bytes_into_fr_repr_safe(hash.as_ref()).into())
     }
 

diff --git a/storage-proofs/core/src/hasher/pedersen.rs b/storage-proofs/core/src/hasher/pedersen.rs
@@ -190,25 +190,33 @@ impl StdHasher for PedersenFunction {
     }
 }
 
+pub fn pedersen_hash<I: IntoIterator<Item = bool>>(
+    node_bits: I,
+) -> fil_sapling_crypto::jubjub::edwards::Point<Bls12, fil_sapling_crypto::jubjub::PrimeOrder> {
+    #[cfg(target_arch = "x86_64")]
+    {
+        use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp;
+        pedersen_hash_bls12_381_with_precomp::<_>(
+            Personalization::None,
+            node_bits,
+            &pedersen::JJ_PARAMS,
+        )
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        use fil_sapling_crypto::pedersen_hash::pedersen_hash;
+        pedersen_hash::<Bls12, _>(Personalization::None, node_bits, &pedersen::JJ_PARAMS)
+    }
+}
+
 impl HashFunction<PedersenDomain> for PedersenFunction {
     fn hash(data: &[u8]) -> PedersenDomain {
         pedersen::pedersen_md_no_padding(data).into()
     }
 
     fn hash2(a: &PedersenDomain, b: &PedersenDomain) -> PedersenDomain {
         let data = NodeBits::new(&(a.0).0[..], &(b.0).0[..]);
-
-        let digest = if cfg!(target_arch = "x86_64") {
-            use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp;
-            pedersen_hash_bls12_381_with_precomp::<_>(
-                Personalization::None,
-                data,
-                &pedersen::JJ_PARAMS,
-            )
-        } else {
-            use fil_sapling_crypto::pedersen_hash::pedersen_hash;
-            pedersen_hash::<Bls12, _>(Personalization::None, data, &pedersen::JJ_PARAMS)
-        };
+        let digest = pedersen_hash(data);
         digest.into_xy().0.into()
     }
 
@@ -319,18 +327,7 @@ impl LightAlgorithm<PedersenDomain> for PedersenFunction {
         _height: usize,
     ) -> PedersenDomain {
         let node_bits = NodeBits::new(&(left.0).0[..], &(right.0).0[..]);
-
-        let digest = if cfg!(target_arch = "x86_64") {
-            use fil_sapling_crypto::pedersen_hash::pedersen_hash_bls12_381_with_precomp;
-            pedersen_hash_bls12_381_with_precomp::<_>(
-                Personalization::None,
-                node_bits,
-                &pedersen::JJ_PARAMS,
-            )
-        } else {
-            use fil_sapling_crypto::pedersen_hash::pedersen_hash;
-            pedersen_hash::<Bls12, _>(Personalization::None, node_bits, &pedersen::JJ_PARAMS)
-        };
+        let digest = pedersen_hash(node_bits);
 
         digest.into_xy().0.into()
     }

diff --git a/storage-proofs/core/src/hasher/sha256.rs b/storage-proofs/core/src/hasher/sha256.rs
@@ -47,7 +47,7 @@ pub struct Sha256Function(Sha256);
 impl StdHasher for Sha256Function {
     #[inline]
     fn write(&mut self, msg: &[u8]) {
-        self.0.input(msg)
+        self.0.update(msg)
     }
 
     #[inline]
@@ -179,7 +179,7 @@ impl HashFunction<Sha256Domain> for Sha256Function {
         let hashed = Sha256::new()
             .chain(AsRef::<[u8]>::as_ref(a))
             .chain(AsRef::<[u8]>::as_ref(b))
-            .result();
+            .finalize();
         let mut res = Sha256Domain::default();
         res.0.copy_from_slice(&hashed[..]);
         res.trim_to_fr32();
@@ -303,7 +303,7 @@ impl Algorithm<Sha256Domain> for Sha256Function {
     #[inline]
     fn hash(&mut self) -> Sha256Domain {
         let mut h = [0u8; 32];
-        h.copy_from_slice(self.0.clone().result().as_ref());
+        h.copy_from_slice(self.0.clone().finalize().as_ref());
         let mut dd = Sha256Domain::from(h);
         dd.trim_to_fr32();
         dd