From bbf87866bd8b742533a3f36121c50f20493b278e Mon Sep 17 00:00:00 2001 From: Artem Storozhuk Date: Thu, 19 Jan 2023 17:08:24 +0200 Subject: [PATCH] Incorporate recent updates to halo2 branch (#1656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use current process binding to limit thread cores (#1633) Use the current processes bound cores to limit the possible cores that threads can be bound to. This allows core binding to work properly when the lotus-worker service is limited to certain CPUs by cgroups. * fix: update ec-gpu-gen (#1638) `ec-gpu-gen` needs to be updated to v0.5 as v0.4 has dependencies that depend on yanked version. It's an indirect dependency of `bellperson` and `neptune`, which are upgraded here. Moving from `memmap` (which is deprecated) to `memmap2` was also needed als dependencies also switched. `chrono` updated their API, so there was also a small change needed. * fix: broken Links in README.md #1637 (#1649) * feat: Introduce PoRepConfig::new_groth16() (#1635) Instead of constructing the `PoRepConfig` directly, use a constructor function. This simplifies the code and makes things less error-prone. Fixes #1632. Co-authored-by: 卜翰 * fix: clean up tree definitions (#1655) There were quite a few public type definitions, that were mostly replaced by the `SectorShape*` types. This commit cleans them up and moves them around if appropriate. This should make the code easier to follow and the public API surface smaller. BREAKING CHANGE: `BinaryLCMerkleTree`, `BinaryMerkleTree`, `BinarySubMerkleTree`, `LCMerkleTree`, `LCStore`, `MerkleStore`, `MerkleTree`, `OctLCMerkleTree`, `OctLCSubMerkleTree`, `OctLCTopMerkleTree`, `OctMerkleTree`, `OctSubMerkleTree`, `OctTopMerkleTree`, `QuadLCMerkleTree` and `QuadMerkleTree` are removed from the public interface. * fix: update ec-gpu-gen (#1638) `ec-gpu-gen` needs to be updated to v0.5 as v0.4 has dependencies that depend on yanked version. It's an indirect dependency of `bellperson` and `neptune`, which are upgraded here. Moving from `memmap` (which is deprecated) to `memmap2` was also needed als dependencies also switched. `chrono` updated their API, so there was also a small change needed. * chore: update Cargo.lock * fix: there was a memmap -> memmap2 missing * fix: make poseidon tests pass Neptune currently is a fork of pasta_curves. That needs patching as well, in order to get the correct names for the fields out. * ci: Apply rustfmt and fix clippy * Pick relevant branch of neptune * fix: Use SHA256 hasher for binary trees Co-authored-by: Clint Armstrong Co-authored-by: Volker Mische Co-authored-by: hanbu97 <98807352+hanbu97@users.noreply.github.com> Co-authored-by: 卜翰 --- Cargo.lock | 757 +++++++++--------- Cargo.toml | 5 +- README.md | 11 +- fil-halo2-gadgets/Cargo.toml | 2 +- fil-halo2-gadgets/src/bin/sha256_compute_k.rs | 4 +- fil-proofs-param/Cargo.toml | 2 +- fil-proofs-param/src/bin/paramcache.rs | 28 +- fil-proofs-tooling/Cargo.toml | 6 +- .../src/bin/benchy/prodbench.rs | 5 +- .../src/bin/circuitinfo/main.rs | 7 +- .../src/bin/update_tree_r_cache/main.rs | 11 +- fil-proofs-tooling/src/metadata.rs | 2 + fil-proofs-tooling/src/shared.rs | 34 +- filecoin-hashers/Cargo.toml | 13 +- filecoin-hashers/src/types.rs | 9 +- filecoin-proofs/Cargo.toml | 6 +- filecoin-proofs/benches/aggregation.rs | 44 +- filecoin-proofs/benches/preprocessing.rs | 17 +- filecoin-proofs/src/api/mod.rs | 2 +- filecoin-proofs/src/api/seal.rs | 12 +- filecoin-proofs/src/constants.rs | 72 +- filecoin-proofs/src/parameters.rs | 6 +- filecoin-proofs/src/types/porep_config.rs | 18 + filecoin-proofs/tests/mod.rs | 41 +- filecoin-proofs/tests/upgrade.rs | 2 +- fr32/Cargo.toml | 2 +- storage-proofs-core/Cargo.toml | 9 +- .../benches/merkle_groth16_halo2.rs | 5 +- storage-proofs-core/src/data.rs | 2 +- storage-proofs-core/src/drgraph.rs | 2 +- storage-proofs-core/src/merkle/builders.rs | 8 +- storage-proofs-core/src/merkle/mod.rs | 46 +- storage-proofs-core/src/merkle/tree.rs | 6 +- storage-proofs-core/src/parameter_cache.rs | 2 +- storage-proofs-core/src/test_helper.rs | 2 +- storage-proofs-porep/Cargo.toml | 7 +- storage-proofs-porep/src/drg/vanilla.rs | 30 +- .../src/stacked/halo2/circuit.rs | 3 +- .../src/stacked/vanilla/cores.rs | 105 ++- .../src/stacked/vanilla/proof.rs | 4 +- storage-proofs-post/Cargo.toml | 4 +- storage-proofs-post/tests/fallback_circuit.rs | 6 +- storage-proofs-update/Cargo.toml | 9 +- storage-proofs-update/src/constants.rs | 3 + storage-proofs-update/src/vanilla.rs | 2 +- 45 files changed, 677 insertions(+), 696 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4430c1a3c..12c243673 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe0133578c0986e1fe3dfcd4af1cc5b2dd6c3dbf534d69916ce16a2701d40ba" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ "cfg-if 1.0.0", "cipher", @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -57,9 +57,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "arrayref" @@ -85,22 +85,22 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" dependencies = [ - "concurrent-queue", + "concurrent-queue 1.2.4", "event-listener", "futures-core", ] [[package]] name = "async-executor" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b" dependencies = [ + "async-lock", "async-task", - "concurrent-queue", + "concurrent-queue 2.0.0", "fastrand", "futures-lite", - "once_cell", "slab", ] @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ "async-channel", "async-executor", @@ -133,31 +133,32 @@ dependencies = [ [[package]] name = "async-io" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" +checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" dependencies = [ + "async-lock", "autocfg", - "concurrent-queue", + "concurrent-queue 2.0.0", "futures-lite", "libc", "log", - "once_cell", "parking", "polling", "slab", "socket2", "waker-fn", - "winapi 0.3.9", + "windows-sys 0.42.0", ] [[package]] name = "async-lock" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" dependencies = [ "event-listener", + "futures-lite", ] [[package]] @@ -174,20 +175,20 @@ dependencies = [ [[package]] name = "async-process" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02111fd8655a613c25069ea89fc8d9bb89331fa77486eb3bc059ee757cfa481c" +checksum = "6381ead98388605d0d9ff86371043b5aa922a3905824244de40dc263a14fcba4" dependencies = [ "async-io", + "async-lock", "autocfg", "blocking", "cfg-if 1.0.0", "event-listener", "futures-lite", "libc", - "once_cell", "signal-hook", - "winapi 0.3.9", + "windows-sys 0.42.0", ] [[package]] @@ -224,9 +225,9 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" dependencies = [ "proc-macro2", "quote", @@ -266,49 +267,44 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.5.4", "object", "rustc-demangle", ] [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bellperson" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b6d7cc52a57ea764d8e9175c6684c6fe8e5d560db78f0f50ccd4ceac3857eb" +checksum = "d1a8623f815c0b1fd89efd9b5f4afbb937f91f51c1ebe3f6dda399c69fa938f3" dependencies = [ "bincode", - "bitvec 0.22.3", - "blake2s_simd 0.5.11", + "blake2s_simd 1.0.0", "blstrs", "byteorder", "crossbeam-channel", - "digest 0.9.0", + "digest 0.10.6", "ec-gpu", "ec-gpu-gen", "ff", "fs2", "group", - "itertools 0.10.5", - "lazy_static", "log", - "memmap", - "num_cpus", + "memmap2", "pairing", "rand", "rand_core", "rayon", "rustversion", "serde", - "sha2 0.9.9", + "sha2 0.10.6", "thiserror", - "yastl", ] [[package]] @@ -357,28 +353,16 @@ dependencies = [ "radium 0.3.0", ] -[[package]] -name = "bitvec" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5237f00a8c86130a0cc317830e558b966dd7850d48a953d998c813f01a41b527" -dependencies = [ - "funty 1.2.0", - "radium 0.6.2", - "tap", - "wyz 0.4.0", -] - [[package]] name = "bitvec" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ - "funty 2.0.0", + "funty", "radium 0.7.0", "tap", - "wyz 0.5.0", + "wyz", ] [[package]] @@ -414,25 +398,13 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding 0.1.5", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] @@ -441,16 +413,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ - "generic-array 0.14.6", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -459,28 +422,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a90ec2df9600c28a01c56c4784c9207a96d2451833aeceb8cc97e4c9548bb78" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] name = "blocking" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" +checksum = "3c67b173a56acffd6d2326fb7ab938ba0b00a71480e14902b2591c87bc5741e8" dependencies = [ "async-channel", + "async-lock", "async-task", "atomic-waker", "fastrand", "futures-lite", - "once_cell", ] [[package]] name = "blst" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c521c26a784d5c4bcd98d483a7d3518376e9ff1efbcfa9e2d456ab8183752303" +checksum = "6a30d0edd9dd1c60ddb42b80341c7852f6f985279a5c1a83659dcb65899dec99" dependencies = [ "cc", "glob", @@ -491,9 +454,9 @@ dependencies = [ [[package]] name = "blstrs" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50ffb24e55817127673bd14f6874ce54b91b338cd0c7d3e4b0da2545f466c459" +checksum = "3ecb6f3a9429706971633edf4b84f922aba9d2e3a7d71bfb450337e64ccb7df0" dependencies = [ "blst", "byte-slice-cast", @@ -520,28 +483,23 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byte-slice-cast" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e" - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "byte-unit" -version = "4.0.14" +version = "4.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ebf10dda65f19ff0f42ea15572a359ed60d7fc74fdc984d90310937be0014b" +checksum = "581ad4b3d627b0c09a0ccb2912148f839acaca0b93cf54cbe42b6c674e86079c" dependencies = [ + "serde", "utf8-width", ] @@ -562,9 +520,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "cache-padded" @@ -589,9 +547,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.73" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" dependencies = [ "jobserver", ] @@ -610,16 +568,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", "js-sys", "num-integer", "num-traits", "serde", - "time 0.1.44", + "time 0.1.45", "wasm-bindgen", "winapi 0.3.9", ] @@ -643,16 +601,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cl3" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a120623848b1af3824734f4f7d8e60e43b9c0cfe86f179a337e383e47234997a" -dependencies = [ - "cl-sys", - "libc", -] - [[package]] name = "cl3" version = "0.6.5" @@ -680,9 +628,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.22" +version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags 1.2.1", @@ -692,7 +640,7 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.15.1", + "textwrap 0.16.0", ] [[package]] @@ -751,6 +699,15 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "concurrent-queue" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.12.0" @@ -902,22 +859,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.11" +version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ "autocfg", "cfg-if 1.0.0", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -925,9 +882,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if 1.0.0", ] @@ -944,7 +901,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.6", + "generic-array", "typenum", ] @@ -972,9 +929,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote", "syn", @@ -1000,9 +957,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f39818dcfc97d45b03953c1292efc4e80954e1583c4aa770bac1383e2310a4" +checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" dependencies = [ "cc", "cxxbridge-flags", @@ -1012,9 +969,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e580d70777c116df50c390d1211993f62d40302881e54d4b79727acb83d0199" +checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" dependencies = [ "cc", "codespan-reporting", @@ -1027,15 +984,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a46460b88d1cec95112c8c363f0e2c39afdb237f60583b0b36343bf627ea9c" +checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" [[package]] name = "cxxbridge-macro" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747b608fecf06b0d72d440f27acc99288207324b793be2c17991839f3d4995ea" +checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" dependencies = [ "proc-macro2", "quote", @@ -1059,44 +1016,25 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer 0.10.3", "crypto-common", ] -[[package]] -name = "dirs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" -dependencies = [ - "cfg-if 0.1.10", - "dirs-sys", -] - [[package]] name = "dirs" version = "4.0.0" @@ -1119,18 +1057,17 @@ dependencies = [ [[package]] name = "ec-gpu" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f1e64cf7ee95dacc8c739e0bf0b06583edaa8e0cee45b27ee2c08ae9343a2e" +checksum = "bd63582de2b59ea1aa48d7c1941b5d87618d95484397521b3acdfa0e1e9f5e45" [[package]] name = "ec-gpu-gen" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1a11b4bfc7a0e48bb2b202a4d3dd2c31cfc9835062229ad332130105f34f82" +checksum = "1f10f085f3d1133b8eab1a3e4fa57efbce689f853e531cca7cbcf4b6a08dca67" dependencies = [ - "bitvec 0.22.3", - "blstrs", + "bitvec 1.0.1", "crossbeam-channel", "ec-gpu", "execute", @@ -1140,11 +1077,9 @@ dependencies = [ "log", "num_cpus", "once_cell", - "pairing", "rayon", - "rust-gpu-tools 0.6.1", + "rust-gpu-tools", "sha2 0.10.6", - "temp-env", "thiserror", "yastl", ] @@ -1225,7 +1160,7 @@ checksum = "313431b1c5e3a6ec9b864333defee57d2ddb50de77abab419e4baedb6cdff292" dependencies = [ "execute-command-macro", "execute-command-tokens", - "generic-array 0.14.6", + "generic-array", ] [[package]] @@ -1302,9 +1237,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df689201f395c6b90dfe87127685f8dbfc083a5e779e613575d8bd7314300c3e" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ "bitvec 1.0.1", "rand_core", @@ -1339,7 +1274,7 @@ dependencies = [ "filecoin-hashers", "filecoin-proofs", "flate2", - "generic-array 0.14.6", + "generic-array", "gperftools", "halo2_proofs", "humansize", @@ -1375,7 +1310,7 @@ dependencies = [ "byte-unit", "bytefmt", "chrono", - "clap 3.2.22", + "clap 3.2.23", "commandspec", "cpu-time", "dialoguer", @@ -1384,13 +1319,13 @@ dependencies = [ "fil_logger", "filecoin-hashers", "filecoin-proofs", - "flexi_logger", - "generic-array 0.14.6", + "flexi_logger 0.22.6", + "generic-array", "git2", "heim", "humansize", "log", - "memmap", + "memmap2", "merkletree", "pasta_curves", "rand", @@ -1423,14 +1358,30 @@ dependencies = [ [[package]] name = "fil_logger" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599d1b83dc76b9db1a1cf1982659e787dc667c358dae3381e806473ba5faf53a" +checksum = "b878f1c1014719d562443a3be956dd85ee550890d4aff6d50ba1d4d7e019694a" dependencies = [ "atty", - "flexi_logger", + "flexi_logger 0.24.1", "log", - "time 0.3.9", +] + +[[package]] +name = "fil_pasta_curves" +version = "0.5.1" +source = "git+https://github.com/filecoin-project/pasta_curves?branch=gpu-fil-fork#016f3ce2486b1f3af330fe86b01bec42aa2d2a44" +dependencies = [ + "blake2b_simd", + "ec-gpu", + "ff", + "group", + "hex", + "lazy_static", + "rand", + "serde", + "static_assertions", + "subtle", ] [[package]] @@ -1444,7 +1395,7 @@ dependencies = [ "ec-gpu", "ff", "fil-halo2-gadgets", - "generic-array 0.14.6", + "generic-array", "halo2_proofs", "hex", "lazy_static", @@ -1472,13 +1423,13 @@ dependencies = [ "fil_logger", "filecoin-hashers", "fr32", - "generic-array 0.14.6", + "generic-array", "gperftools", "halo2_proofs", "hex", "lazy_static", "log", - "memmap", + "memmap2", "merkletree", "once_cell", "rand", @@ -1498,24 +1449,24 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" +checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.6.2", ] [[package]] @@ -1535,6 +1486,23 @@ dependencies = [ "time 0.3.9", ] +[[package]] +name = "flexi_logger" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99659bcfd52cfece972bd00acb9dba7028094d47e699ea8b193b9aaebd5c362b" +dependencies = [ + "ansi_term", + "atty", + "chrono", + "glob", + "lazy_static", + "log", + "regex", + "rustversion", + "thiserror", +] + [[package]] name = "flume" version = "0.10.14" @@ -1603,12 +1571,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "funty" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1847abb9cb65d566acd5942e94aea9c8f547ad02c98e1649326fc0e8910b8b1e" - [[package]] name = "funty" version = "2.0.0" @@ -1617,9 +1579,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -1631,9 +1593,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -1641,15 +1603,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-lite" @@ -1668,15 +1630,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-timer" @@ -1686,9 +1648,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -1701,15 +1663,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.6" @@ -1722,9 +1675,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1783,11 +1736,10 @@ dependencies = [ [[package]] name = "group" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7391856def869c1c81063a03457c676fbcd419709c3dfb33d8d319de484b154d" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ - "byteorder", "ff", "rand", "rand_core", @@ -1797,9 +1749,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ "bytes", "fnv", @@ -1841,11 +1793,14 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.2.0" -source = "git+https://github.com/filecoin-project/halo2?branch=serialize-vk-pk#7a7d8a177b16632c7271dde0db8ec810c3be6ae5" +source = "git+https://github.com/filecoin-project/halo2?branch=halo2#11bb390954ab8181ebfaed6e7c0186e5f52332ef" dependencies = [ "blake2b_simd", + "ec-gpu", + "ec-gpu-gen", "ff", "group", + "log", "pasta_curves", "rand_core", "rayon", @@ -1977,6 +1932,9 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "http" @@ -2035,9 +1993,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes", "futures-channel", @@ -2072,9 +2030,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.51" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2086,9 +2044,9 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde6edd6cef363e9359ed3c98ba64590ba9eecba2293eb5a723ab32aee8926aa" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" dependencies = [ "cxx", "cxx-build", @@ -2106,9 +2064,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -2132,8 +2090,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "block-padding 0.3.2", - "generic-array 0.14.6", + "block-padding", + "generic-array", ] [[package]] @@ -2147,9 +2105,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" [[package]] name = "itertools" @@ -2226,9 +2184,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.135" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "libgit2-sys" @@ -2315,29 +2273,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] -name = "memmap" -version = "0.7.0" +name = "memmap2" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" dependencies = [ "libc", - "winapi 0.3.9", ] [[package]] -name = "memmap2" -version = "0.5.7" +name = "memoffset" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "libc", + "autocfg", ] [[package]] name = "memoffset" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" dependencies = [ "autocfg", ] @@ -2380,23 +2337,32 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "native-tls" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -2412,8 +2378,8 @@ dependencies = [ [[package]] name = "neptune" -version = "7.2.0" -source = "git+https://github.com/filecoin-project/neptune?branch=halo2#fb744540b9bba5cd6064ebfaa259603587d72283" +version = "8.1.0" +source = "git+https://github.com/filecoin-project/neptune?branch=halo2#6daf50d6bc88bbf9ef9a12c816ebca6d12478d0d" dependencies = [ "bellperson", "blake2s_simd 0.5.11", @@ -2421,17 +2387,15 @@ dependencies = [ "byteorder", "ec-gpu", "ec-gpu-gen", - "execute", "ff", - "generic-array 0.14.6", + "fil_pasta_curves", + "generic-array", "halo2_proofs", - "hex", "itertools 0.8.2", "lazy_static", "log", - "pasta_curves", - "rust-gpu-tools 0.5.0", - "sha2 0.9.9", + "serde", + "trait-set", ] [[package]] @@ -2470,7 +2434,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -2557,9 +2521,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", @@ -2591,9 +2555,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "oorandom" @@ -2601,43 +2565,27 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "opencl3" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8862f86c2b3f757038243318edb55a47b1be7d46376fe6bdd9aedd1b0074902" -dependencies = [ - "cl3 0.4.4", - "libc", -] - [[package]] name = "opencl3" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "931cc2ab3068142384dbdaba142681c11b315cf3b96c7a59e8480d062363387f" dependencies = [ - "cl3 0.6.5", + "cl3", "libc", ] [[package]] name = "openssl" -version = "0.10.42" +version = "0.10.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" dependencies = [ "bitflags 1.2.1", "cfg-if 1.0.0", @@ -2667,18 +2615,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.22.0+1.1.1q" +version = "111.24.0+1.1.1s" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.76" +version = "0.9.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" dependencies = [ "autocfg", "cc", @@ -2690,9 +2638,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.0" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "output_vt100" @@ -2721,8 +2669,7 @@ checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" [[package]] name = "pasta_curves" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "369d7785168ad7ff0cbe467d968ca3e19a927d8536b11ef9c21b4e454b15ba42" +source = "git+https://github.com/filecoin-project/pasta_curves?branch=gpu#f976b78c799e77430ddc7047b265163381897da0" dependencies = [ "blake2b_simd", "ec-gpu", @@ -2748,7 +2695,7 @@ checksum = "ff5751d87f7c00ae6403eb1fcbba229b9c76c9a30de8c1cf87182177b168cea2" dependencies = [ "crossbeam-channel", "libc", - "time 0.1.44", + "time 0.1.45", "winapi 0.3.9", ] @@ -2772,9 +2719,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "platforms" @@ -2812,23 +2759,23 @@ dependencies = [ [[package]] name = "polling" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" +checksum = "9f7d73f1eaed1ca1fb37b54dcc9b38e3b17d6c7b8ecb7abfffcac8d0351f17d4" dependencies = [ "autocfg", "cfg-if 1.0.0", "libc", "log", "wepoll-ffi", - "winapi 0.3.9", + "windows-sys 0.42.0", ] [[package]] name = "positioned-io" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "173e3686d2cec75681bebc1d84d8ed1e22a0336ccde102ac1d2d134679173ad8" +checksum = "09b9485cf7f528baf34edd811ec8283a168864912e11d0b7d3e0510738761114" dependencies = [ "byteorder", "libc", @@ -2837,9 +2784,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pretty_assertions" @@ -2879,9 +2826,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -2933,12 +2880,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac" -[[package]] -name = "radium" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" - [[package]] name = "radium" version = "0.7.0" @@ -2995,11 +2936,10 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" dependencies = [ - "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -3007,9 +2947,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.3" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -3039,9 +2979,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -3056,9 +2996,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -3071,9 +3011,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64", "bytes", @@ -3118,35 +3058,19 @@ dependencies = [ "tempfile", ] -[[package]] -name = "rust-gpu-tools" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1785ab2a8accec77189e23fead221f2543cebf7ccf569788511cdac9f5fad168" -dependencies = [ - "dirs 2.0.2", - "fil-rustacuda", - "hex", - "lazy_static", - "log", - "opencl3 0.4.1", - "sha2 0.8.2", - "thiserror", -] - [[package]] name = "rust-gpu-tools" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b2838e99bd4c9b3e6a963194440c6c5b66721d3f127625d709236ecaa1a730f" dependencies = [ - "dirs 4.0.0", + "dirs", "fil-rustacuda", "hex", "lazy_static", "log", "once_cell", - "opencl3 0.6.3", + "opencl3", "sha2 0.10.6", "temp-env", "thiserror", @@ -3215,7 +3139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -3261,9 +3185,9 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.145" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] @@ -3280,9 +3204,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -3291,9 +3215,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" dependencies = [ "itoa 1.0.4", "ryu", @@ -3312,18 +3236,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha2" version = "0.9.9" @@ -3334,7 +3246,7 @@ dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -3345,7 +3257,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", "sha2-asm", ] @@ -3364,10 +3276,10 @@ version = "7.0.0" dependencies = [ "byteorder", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", "fake-simd", "lazy_static", - "opaque-debug 0.3.0", + "opaque-debug", "rand", "rand_xorshift", "sha2 0.10.6", @@ -3472,13 +3384,13 @@ dependencies = [ "filecoin-hashers", "fr32", "fs2", - "generic-array 0.14.6", + "generic-array", "gperftools", "halo2_proofs", "itertools 0.10.5", "lazy_static", "log", - "memmap", + "memmap2", "merkletree", "num_cpus", "pretty_assertions", @@ -3514,7 +3426,7 @@ dependencies = [ "fil_logger", "filecoin-hashers", "fr32", - "generic-array 0.14.6", + "generic-array", "glob", "halo2_proofs", "hex", @@ -3556,7 +3468,7 @@ dependencies = [ "fil_logger", "filecoin-hashers", "fr32", - "generic-array 0.14.6", + "generic-array", "halo2_proofs", "hex", "log", @@ -3582,11 +3494,11 @@ dependencies = [ "fil_logger", "filecoin-hashers", "fr32", - "generic-array 0.14.6", + "generic-array", "halo2_proofs", "lazy_static", "log", - "memmap", + "memmap2", "merkletree", "neptune", "rand", @@ -3643,9 +3555,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", @@ -3734,9 +3646,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" @@ -3769,9 +3681,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -3823,9 +3735,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", "bytes", @@ -3909,6 +3821,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trait-set" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79e2e9c9ab44c6d7c20d5976961b47e8f49ac199154daa514b77cd1ab536625" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "traitobject" version = "0.1.0" @@ -4251,43 +4174,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.10.1" @@ -4299,18 +4279,9 @@ dependencies = [ [[package]] name = "wyz" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "129e027ad65ce1453680623c3fb5163cbf7107bfe1aa32257e7d0e63f9ced188" -dependencies = [ - "tap", -] - -[[package]] -name = "wyz" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ "tap", ] diff --git a/Cargo.toml b/Cargo.toml index d34aa8edd..fe2d719bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,4 +15,7 @@ members = [ ] [patch.crates-io] -halo2_proofs = { git = "https://github.com/filecoin-project/halo2", branch = "serialize-vk-pk" } +halo2_proofs = { git = "https://github.com/filecoin-project/halo2", branch = "halo2" } +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "halo2" } +pasta_curves = { git = "https://github.com/filecoin-project/pasta_curves", branch = "gpu" } +fil_pasta_curves = { git = "https://github.com/filecoin-project/pasta_curves", branch = "gpu-fil-fork" } diff --git a/README.md b/README.md index 0ff924605..f33c2ff97 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,10 @@ The **Filecoin Proving Subsystem** (or FPS) provides the storage proofs required There are currently several different crates: -- [**Storage Proofs (`storage-proofs`)**](./storage-proofs) - A library for constructing storage proofs – including non-circuit proofs, corresponding SNARK circuits, and a method of combining them. - -- [**Storage Proofs Core (`storage-proofs-core`)**](./storage-proofs/core) +- [**Storage Proofs Core (`storage-proofs-core`)**](./storage-proofs-core) A set of common primitives used throughout the other storage-proofs sub-crates, including crypto, merkle tree, hashing and gadget interfaces. -- [**Storage Proofs PoRep (`storage-proofs-porep`)**](./storage-proofs/porep) +- [**Storage Proofs PoRep (`storage-proofs-porep`)**](./storage-proofs-porep) `storage-proofs-porep` is intended to serve as a reference implementation for _**Proof-of-Replication**_ (**PoRep**), while also performing the heavy lifting for `filecoin-proofs`. Primary Components: @@ -18,7 +15,7 @@ There are currently several different crates: - **DrgPoRep** (_Depth Robust Graph_ **_Proof-of-Replication_**) - **StackedDrgPoRep** -- [**Storage Proofs PoSt (`storage-proofs-post`)**](./storage-proofs/post) +- [**Storage Proofs PoSt (`storage-proofs-post`)**](./storage-proofs-post) `storage-proofs-post` is intended to serve as a reference implementation for _**Proof-of-Space-time**_ (**PoSt**), for `filecoin-proofs`. Primary Components: @@ -30,7 +27,7 @@ There are currently several different crates: ## Security Audits -The `rust-fil-proofs` proofs code and the [Filecoin Spec](https://bafybeidxw5vxjdwsun2zc2illagf43v6w5r5w63vg455h7vjesbyqssg64.ipfs.dweb.link/algorithms/sdr/) has undergone a [proofs security audit](audits/Sigma-Prime-Protocol-Labs-Filecoin-Proofs-Security-Review-v2.1.pdf) performed by [Sigma Prime](https://sigmaprime.io/) and been deemed free of *critical* or *major* security issues. In addition to the security review, the document provides the summary of findings, vulnerability classifications, and recommended resolutions. All known issues have been resolved to date in both the code and the specification. +The `rust-fil-proofs` proofs code and the [Filecoin Spec](https://spec.filecoin.io/algorithms/sdr/) has undergone a [proofs security audit](audits/Sigma-Prime-Protocol-Labs-Filecoin-Proofs-Security-Review-v2.1.pdf) performed by [Sigma Prime](https://sigmaprime.io/) and been deemed free of *critical* or *major* security issues. In addition to the security review, the document provides the summary of findings, vulnerability classifications, and recommended resolutions. All known issues have been resolved to date in both the code and the specification. `rust-fil-proofs` has also undergone a [SNARK proofs security audit performed by Dr. Jean-Philippe Aumasson and Antony Vennard](audits/protocolai-audit-20200728.pdf) and been deemed free of *critical* or *major* security issues. In addition to the security analysis, the document provides the audit goals, methodology, functionality descriptions and finally observations on what could be improved. All known issues have been resolved to date. diff --git a/fil-halo2-gadgets/Cargo.toml b/fil-halo2-gadgets/Cargo.toml index a0cfc916c..4bb6bb4a1 100644 --- a/fil-halo2-gadgets/Cargo.toml +++ b/fil-halo2-gadgets/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" ff = "0.12.0" halo2_proofs = "0.2.0" halo2_gadgets = "0.2.0" -neptune = { git = "https://github.com/filecoin-project/neptune", branch = "halo2" } +neptune = "~8.1.0" sha2 = "0.9.2" [dev-dependencies] diff --git a/fil-halo2-gadgets/src/bin/sha256_compute_k.rs b/fil-halo2-gadgets/src/bin/sha256_compute_k.rs index 66ae6fe6f..20ee3ef03 100644 --- a/fil-halo2-gadgets/src/bin/sha256_compute_k.rs +++ b/fil-halo2-gadgets/src/bin/sha256_compute_k.rs @@ -131,7 +131,7 @@ fn sha256_chip_compute_k() { Ok(_) => { // Proof size does not change with circuit size. if num_blocks == PREIMAGE_BLOCK_SIZES.start { - let cost = CircuitCost::::measure(k as usize, &circ); + let cost = CircuitCost::::measure(k, &circ); let proof_size: usize = cost.marginal_proof_size().into(); println!("\tproof_size = {} bytes", proof_size); } @@ -219,7 +219,7 @@ fn compress_chip_compute_k() { Ok(_) => { // Proof size does not change with circuit size. if num_blocks == PREIMAGE_BLOCK_SIZES.start { - let cost = CircuitCost::::measure(k as usize, &circ); + let cost = CircuitCost::::measure(k, &circ); let proof_size: usize = cost.marginal_proof_size().into(); println!("\tproof_size = {} bytes", proof_size); } diff --git a/fil-proofs-param/Cargo.toml b/fil-proofs-param/Cargo.toml index 0a6ede670..be29572c6 100644 --- a/fil-proofs-param/Cargo.toml +++ b/fil-proofs-param/Cargo.toml @@ -32,7 +32,7 @@ structopt = "0.3.12" humansize = "1.1.0" indicatif = "0.16.2" dialoguer = "0.10.0" -blstrs = "0.5.0" +blstrs = "0.6.1" typenum = "1.11.2" generic-array = "0.14.4" halo2_proofs = "0.2.0" diff --git a/fil-proofs-param/src/bin/paramcache.rs b/fil-proofs-param/src/bin/paramcache.rs index 10295c9c2..15ef5b783 100644 --- a/fil-proofs-param/src/bin/paramcache.rs +++ b/fil-proofs-param/src/bin/paramcache.rs @@ -6,7 +6,7 @@ use blstrs::Scalar as Fr; use dialoguer::{theme::ColorfulTheme, MultiSelect}; use filecoin_proofs::{ constants::{ - DefaultPieceHasher, POREP_PARTITIONS, PUBLISHED_SECTOR_SIZES, WINDOW_POST_CHALLENGE_COUNT, + DefaultPieceHasher, PUBLISHED_SECTOR_SIZES, WINDOW_POST_CHALLENGE_COUNT, WINDOW_POST_SECTOR_COUNT, WINNING_POST_CHALLENGE_COUNT, WINNING_POST_SECTOR_COUNT, }, parameters::{public_params, window_post_public_params, winning_post_public_params}, @@ -226,18 +226,7 @@ fn generate_params_porep(sector_size: u64, api_version: ApiVersion) { sector_size, Fr, cache_porep_params, - PoRepConfig { - sector_size: SectorSize(sector_size), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(§or_size) - .expect("unknown sector size") as usize, - ), - porep_id: [0; 32], - api_version, - } + PoRepConfig::new_groth16(sector_size, [0; 32], api_version) ); } @@ -246,18 +235,7 @@ fn generate_params_empty_sector_update(sector_size: u64, api_version: ApiVersion sector_size, Fr, cache_empty_sector_update_params, - PoRepConfig { - sector_size: SectorSize(sector_size), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(§or_size) - .expect("unknown sector size") as usize, - ), - porep_id: [0; 32], - api_version, - } + PoRepConfig::new_groth16(sector_size, [0; 32], api_version) ); } diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index 955e0c8b7..b1675ed6b 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -21,8 +21,8 @@ serde_json = "1.0" regex = "1.3.7" commandspec = "0.12.2" chrono = { version = "0.4.7", features = ["serde"] } -memmap = "0.7.0" -bellperson = "0.22.0" +memmap2 = "0.5.6" +bellperson = "0.24.0" rand = "0.8" tempfile = "3.0.8" cpu-time = "1.0.0" @@ -47,7 +47,7 @@ fdlimit = "0.2.0" dialoguer = "0.10.0" structopt = "0.3.12" humansize = "1.1.0" -blstrs = "0.5.0" +blstrs = "0.6.1" time = "0.3.9" pasta_curves = "0.4.0" diff --git a/fil-proofs-tooling/src/bin/benchy/prodbench.rs b/fil-proofs-tooling/src/bin/benchy/prodbench.rs index 4af528003..45c6aeb5c 100644 --- a/fil-proofs-tooling/src/bin/benchy/prodbench.rs +++ b/fil-proofs-tooling/src/bin/benchy/prodbench.rs @@ -170,9 +170,8 @@ fn configure_global_config(inputs: &ProdbenchInputs) { .expect("POREP_PARTITIONS poisoned") .insert(inputs.sector_size_bytes(), inputs.porep_partitions); POREP_MINIMUM_CHALLENGES - .write() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .insert(inputs.sector_size_bytes(), inputs.porep_challenges); + .get_mut() + .insert(inputs.sector_size_bytes(), inputs.porep_challenges as usize); } pub fn run( diff --git a/fil-proofs-tooling/src/bin/circuitinfo/main.rs b/fil-proofs-tooling/src/bin/circuitinfo/main.rs index 98d5e0f53..657cb9f27 100644 --- a/fil-proofs-tooling/src/bin/circuitinfo/main.rs +++ b/fil-proofs-tooling/src/bin/circuitinfo/main.rs @@ -159,12 +159,7 @@ fn porep_info(sector_size: u64, api_version: ApiVersion) -> (CircuitInfo, usize) sector_size, Fr, get_porep_info, - PoRepConfig { - sector_size: SectorSize(sector_size), - partitions, - porep_id: [0; 32], - api_version, - } + PoRepConfig::new_groth16(sector_size, [0; 32], api_version) ); (info, partitions.into()) } diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index 765741ace..10aeee5d6 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -13,17 +13,16 @@ use filecoin_proofs::{ SectorShapeSub2, SectorShapeSub8, SectorShapeTop2, OCT_ARITY, }; use generic_array::typenum::Unsigned; -use memmap::MmapOptions; +use memmap2::MmapOptions; use merkletree::{ merkle::get_merkle_tree_len, - store::{ExternalReader, ReplicaConfig, Store, StoreConfig}, + store::{ExternalReader, LevelCacheStore, ReplicaConfig, Store, StoreConfig}, }; use pasta_curves::{Fp, Fq}; use storage_proofs_core::{ cache_key::CacheKey, merkle::{ - create_lc_tree, get_base_tree_count, split_config_and_replica, LCStore, LCTree, - MerkleTreeTrait, + create_lc_tree, get_base_tree_count, split_config_and_replica, LCTree, MerkleTreeTrait, }, util::{default_rows_to_discard, NODE_SIZE}, }; @@ -75,7 +74,7 @@ where let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; let tree_r_last_root = if is_sector_shape_base(sector_size) { ensure!(configs.len() == 1, "Invalid tree-shape specified"); - let store = LCStore::>::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, OCT_ARITY, &configs[0], @@ -255,7 +254,7 @@ where // First, read the roots from the cached trees on disk let mut cached_base_tree_roots: Vec> = Vec::with_capacity(tree_count); for (i, config) in configs.iter().enumerate().take(tree_count) { - let store = LCStore::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, OCT_ARITY, config, diff --git a/fil-proofs-tooling/src/metadata.rs b/fil-proofs-tooling/src/metadata.rs index b91667975..40c1ca995 100644 --- a/fil-proofs-tooling/src/metadata.rs +++ b/fil-proofs-tooling/src/metadata.rs @@ -40,6 +40,8 @@ impl GitMetadata { let repo = Repository::discover(&repo_path)?; let head = repo.head()?; let commit = head.peel_to_commit()?; + + // Unwrap is OK as the given seconds won't be out of range. let date = Utc.timestamp_opt(commit.time().seconds(), 0).unwrap(); Ok(GitMetadata { diff --git a/fil-proofs-tooling/src/shared.rs b/fil-proofs-tooling/src/shared.rs index e9a4807a8..d54191e2d 100644 --- a/fil-proofs-tooling/src/shared.rs +++ b/fil-proofs-tooling/src/shared.rs @@ -7,10 +7,10 @@ use filecoin_hashers::Hasher; use filecoin_proofs::{ add_piece, fauxrep_aux, seal_pre_commit_phase1, seal_pre_commit_phase2, validate_cache_for_precommit_phase2, DefaultPieceHasher, DefaultTreeHasher, MerkleTreeTrait, - PaddedBytesAmount, PieceInfo, PoRepConfig, PoRepProofPartitions, PoStConfig, PoStType, - PrivateReplicaInfo, PublicReplicaInfo, SealPreCommitOutput, SealPreCommitPhase1Output, - SectorSize, UnpaddedBytesAmount, POREP_PARTITIONS, WINDOW_POST_CHALLENGE_COUNT, - WINDOW_POST_SECTOR_COUNT, WINNING_POST_CHALLENGE_COUNT, WINNING_POST_SECTOR_COUNT, + PaddedBytesAmount, PieceInfo, PoRepConfig, PoStConfig, PoStType, PrivateReplicaInfo, + PublicReplicaInfo, SealPreCommitOutput, SealPreCommitPhase1Output, SectorSize, + UnpaddedBytesAmount, WINDOW_POST_CHALLENGE_COUNT, WINDOW_POST_SECTOR_COUNT, + WINNING_POST_CHALLENGE_COUNT, WINNING_POST_SECTOR_COUNT, }; use generic_array::typenum::Unsigned; use log::info; @@ -137,18 +137,7 @@ where let sector_size_unpadded_bytes_ammount = UnpaddedBytesAmount::from(PaddedBytesAmount::from(sector_size)); - let porep_config = PoRepConfig { - sector_size, - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("poisoned read access") - .get(&u64::from(sector_size)) - .expect("unknown sector size") as usize, - ), - porep_id, - api_version, - }; + let porep_config = PoRepConfig::new_groth16(u64::from(sector_size), porep_id, api_version); let mut out: Vec<(SectorId, PreCommitReplicaOutput)> = Default::default(); let mut sector_ids = Vec::new(); @@ -343,18 +332,7 @@ pub fn get_porep_config(sector_size: u64, api_version: ApiVersion let arbitrary_porep_id = [99; 32]; if util::is_groth16_field::() { - PoRepConfig { - sector_size: sector_size.into(), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITONS poisoned") - .get(&(sector_size)) - .expect("unknown sector size") as usize, - ), - porep_id: arbitrary_porep_id, - api_version, - } + PoRepConfig::new_groth16(sector_size, arbitrary_porep_id, api_version) } else { PoRepConfig::new_halo2(sector_size.into(), arbitrary_porep_id, api_version) } diff --git a/filecoin-hashers/Cargo.toml b/filecoin-hashers/Cargo.toml index d50d93831..a0cb4dfca 100644 --- a/filecoin-hashers/Cargo.toml +++ b/filecoin-hashers/Cargo.toml @@ -9,17 +9,16 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] -bellperson = "0.22.0" -blstrs = "0.5.0" +bellperson = "0.24.0" +blstrs = "0.6.1" generic-array = "0.14.4" merkletree = "0.22.0" -ec-gpu = "0.1.0" +ec-gpu = "0.2.0" ff = "0.12.0" anyhow = "1.0.34" serde = "1.0.117" rand = "0.8.0" -# neptune = { version = "~7.0.0", optional = true, features = ["bls", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } -neptune = { git = "https://github.com/filecoin-project/neptune", branch = "halo2", optional = true, features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { version = "~8.1.0", optional = true, features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } lazy_static = { version = "1.4.0", optional = true } blake2s_simd = { version = "1.0.0", optional = true } sha2 = { version = "0.10.2", optional = true } @@ -31,8 +30,8 @@ fil-halo2-gadgets = { path = "../fil-halo2-gadgets" } [features] default = ["opencl", "blake2s", "poseidon", "sha256"] -cuda = ["bellperson/cuda", "neptune/cuda"] -opencl = ["bellperson/opencl", "neptune/opencl"] +cuda = ["bellperson/cuda", "halo2_proofs/cuda", "neptune/cuda"] +opencl = ["bellperson/opencl", "halo2_proofs/opencl", "neptune/opencl"] # available hashers blake2s = ["blake2s_simd"] diff --git a/filecoin-hashers/src/types.rs b/filecoin-hashers/src/types.rs index 57cb695fc..2a415c616 100644 --- a/filecoin-hashers/src/types.rs +++ b/filecoin-hashers/src/types.rs @@ -10,7 +10,6 @@ use bellperson::{ ConstraintSystem, SynthesisError, }; use blstrs::Scalar as Fr; -use ec_gpu::GpuField; use ff::{Field, PrimeField}; use fil_halo2_gadgets::ColumnCount; use halo2_proofs::{ @@ -44,7 +43,7 @@ pub trait Domain: + Element + StdHash { - type Field: PrimeField + GpuField; + type Field: PrimeField; #[allow(clippy::wrong_self_convention)] fn into_bytes(&self) -> Vec { @@ -99,7 +98,11 @@ pub trait HashFunction: Clone + Debug + Send + Sync + LightAlgorithm< } pub trait Hasher: Clone + Debug + Eq + Default + Send + Sync { - type Field: PrimeField + GpuField; + #[cfg(not(any(feature = "cuda", feature = "opencl")))] + type Field: PrimeField; + #[cfg(any(feature = "cuda", feature = "opencl"))] + type Field: PrimeField + ec_gpu::GpuName; + type Domain: Domain + LightHashable + AsRef; type Function: HashFunction; diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 913aa1e4b..bb96cd006 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -16,12 +16,12 @@ storage-proofs-update = { path = "../storage-proofs-update", version = "~12.0.0" filecoin-hashers = { version = "~7.0.0", path = "../filecoin-hashers", default-features = false, features = ["poseidon", "sha256"] } rand = "0.8" lazy_static = "1.2" -memmap = "0.7" +memmap2 = "0.5.6" serde = { version = "1.0", features = ["rc", "derive"] } serde_json = "1.0" ff = "0.12.0" blake2b_simd = "1.0.0" -bellperson = "0.22.0" +bellperson = "0.24.0" log = "0.4.7" rayon = "1.1.0" hex = "0.4.0" @@ -34,7 +34,7 @@ gperftools = { version = "0.2", optional = true } generic-array = "0.14.4" fr32 = { path = "../fr32", version = "~5.0.0", default-features = false } once_cell = "1.8.0" -blstrs = "0.5.0" +blstrs = "0.6.1" halo2_proofs = "0.2.0" [dev-dependencies] diff --git a/filecoin-proofs/benches/aggregation.rs b/filecoin-proofs/benches/aggregation.rs index 67b270055..edf544bfd 100644 --- a/filecoin-proofs/benches/aggregation.rs +++ b/filecoin-proofs/benches/aggregation.rs @@ -5,8 +5,8 @@ use blstrs::Scalar as Fr; use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; use filecoin_proofs::{ caches::{get_stacked_srs_key, get_stacked_srs_verifier_key}, - get_seal_inputs, PoRepConfig, PoRepProofPartitions, SectorShape2KiB, SectorShape32GiB, - SectorSize, POREP_PARTITIONS, SECTOR_SIZE_2_KIB, SECTOR_SIZE_32_GIB, + get_seal_inputs, PoRepConfig, SectorShape2KiB, SectorShape32GiB, SECTOR_SIZE_2_KIB, + SECTOR_SIZE_32_GIB, }; use rand::{thread_rng, Rng}; use storage_proofs_core::{api_version::ApiVersion, is_legacy_porep_id}; @@ -29,18 +29,8 @@ fn bench_seal_inputs(c: &mut Criterion) { porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); assert!(!is_legacy_porep_id(porep_id)); - let config = PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_2_KIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_2_KIB) - .expect("unknown sector size") as usize, - ), - porep_id, - api_version: ApiVersion::V1_1_0, - }; + let config = PoRepConfig::new_groth16(SECTOR_SIZE_2_KIB, porep_id, ApiVersion::V1_1_0); + let comm_r = [5u8; 32]; let comm_d = [6u8; 32]; let prover_id = [7u8; 32]; @@ -80,18 +70,7 @@ fn bench_stacked_srs_key(c: &mut Criterion) { porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); assert!(!is_legacy_porep_id(porep_id)); - let config = PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_32_GIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_32_GIB) - .expect("unknown sector size") as usize, - ), - porep_id, - api_version: ApiVersion::V1_1_0, - }; + let config = PoRepConfig::new_groth16(SECTOR_SIZE_32_GIB, porep_id, ApiVersion::V1_1_0); let mut group = c.benchmark_group("bench-stacked-srs-key"); for num_proofs_to_aggregate in params { @@ -124,18 +103,7 @@ fn bench_stacked_srs_verifier_key(c: &mut Criterion) { porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); assert!(!is_legacy_porep_id(porep_id)); - let config = PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_32_GIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_32_GIB) - .expect("unknown sector size") as usize, - ), - porep_id, - api_version: ApiVersion::V1_1_0, - }; + let config = PoRepConfig::new_groth16(SECTOR_SIZE_32_GIB, porep_id, ApiVersion::V1_1_0); let mut group = c.benchmark_group("bench-stacked-srs-verifier-key"); for num_proofs_to_aggregate in params { diff --git a/filecoin-proofs/benches/preprocessing.rs b/filecoin-proofs/benches/preprocessing.rs index e71986080..d5afa380f 100644 --- a/filecoin-proofs/benches/preprocessing.rs +++ b/filecoin-proofs/benches/preprocessing.rs @@ -4,8 +4,8 @@ use std::time::Duration; use blstrs::Scalar as Fr; use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use filecoin_proofs::{ - add_piece, get_seal_inputs, PaddedBytesAmount, PoRepConfig, PoRepProofPartitions, - SectorShape2KiB, SectorSize, UnpaddedBytesAmount, POREP_PARTITIONS, SECTOR_SIZE_2_KIB, + add_piece, get_seal_inputs, PaddedBytesAmount, PoRepConfig, SectorShape2KiB, + UnpaddedBytesAmount, SECTOR_SIZE_2_KIB, }; use fr32::Fr32Reader; use rand::{thread_rng, Rng}; @@ -115,18 +115,7 @@ fn get_seal_inputs_benchmark(c: &mut Criterion) { porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes()); assert!(!is_legacy_porep_id(porep_id)); - let config = PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_2_KIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_2_KIB) - .expect("unknown sector size") as usize, - ), - porep_id, - api_version: ApiVersion::V1_1_0, - }; + let config = PoRepConfig::new_groth16(SECTOR_SIZE_2_KIB, porep_id, ApiVersion::V1_1_0); let comm_r: [u8; 32] = [5u8; 32]; let comm_d: [u8; 32] = [6u8; 32]; let prover_id: [u8; 32] = [7u8; 32]; diff --git a/filecoin-proofs/src/api/mod.rs b/filecoin-proofs/src/api/mod.rs index 5d6dda51a..77d37925f 100644 --- a/filecoin-proofs/src/api/mod.rs +++ b/filecoin-proofs/src/api/mod.rs @@ -9,7 +9,7 @@ use filecoin_hashers::Hasher; use fr32::{write_unpadded, Fr32Reader}; use generic_array::typenum::{U2, U8}; use log::{info, trace}; -use memmap::MmapOptions; +use memmap2::MmapOptions; use merkletree::store::{DiskStore, LevelCacheStore, StoreConfig}; use storage_proofs_core::{ cache_key::CacheKey, diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index 107e2246d..4983651cf 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -11,7 +11,7 @@ use ff::PrimeField; use filecoin_hashers::{Domain, Groth16Hasher, Hasher, PoseidonArity}; use halo2_proofs::pasta::{Fp, Fq}; use log::{info, trace}; -use memmap::MmapOptions; +use memmap2::MmapOptions; use merkletree::store::{DiskStore, Store, StoreConfig}; use rayon::prelude::*; use sha2::{Digest, Sha256}; @@ -36,6 +36,7 @@ use storage_proofs_porep::stacked::{ StackedDrg, Tau, TemporaryAux, TemporaryAuxCache, }; +use crate::POREP_MINIMUM_CHALLENGES; use crate::{ api::{ as_safe_commitment, commitment_from_fr, get_base_tree_leafs, get_base_tree_size, @@ -47,7 +48,7 @@ use crate::{ }, constants::{ DefaultBinaryTree, DefaultPieceDomain, DefaultPieceHasher, DefaultTreeDomain, - DefaultTreeHasher, POREP_MINIMUM_CHALLENGES, SINGLE_PARTITION_PROOF_LEN, + DefaultTreeHasher, SINGLE_PARTITION_PROOF_LEN, }, parameters::setup_params, pieces::{self, verify_pieces}, @@ -1944,11 +1945,8 @@ where &public_inputs, &proof, &ChallengeRequirements { - minimum_challenges: *POREP_MINIMUM_CHALLENGES - .read() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .get(&u64::from(SectorSize::from(porep_config))) - .expect("unknown sector size") as usize, + minimum_challenges: POREP_MINIMUM_CHALLENGES + .from_sector_size(u64::from(SectorSize::from(porep_config))), }, ) } diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index dd2aec07b..b22eac962 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -1,5 +1,5 @@ -use std::collections::HashMap; use std::sync::RwLock; +use std::{collections::HashMap, sync::RwLockWriteGuard}; pub use storage_proofs_core::drgraph::BASE_DEGREE as DRG_DEGREE; pub use storage_proofs_porep::stacked::EXP_DEGREE; @@ -7,7 +7,7 @@ pub use storage_proofs_porep::stacked::EXP_DEGREE; use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher, Hasher}; use lazy_static::lazy_static; use storage_proofs_core::{ - merkle::{BinaryMerkleTree, LCTree, OctLCMerkleTree, OctMerkleTree}, + merkle::{BinaryMerkleTree, DiskTree, LCTree}, util::NODE_SIZE, MAX_LEGACY_POREP_REGISTERED_PROOF_ID, }; @@ -47,24 +47,47 @@ pub const PUBLISHED_SECTOR_SIZES: [u64; 10] = [ SECTOR_SIZE_64_GIB, ]; +pub struct PorepMinimumChallenges(RwLock>); +impl PorepMinimumChallenges { + fn new() -> Self { + Self(RwLock::new( + [ + (SECTOR_SIZE_2_KIB, 2), + (SECTOR_SIZE_4_KIB, 2), + (SECTOR_SIZE_16_KIB, 2), + (SECTOR_SIZE_32_KIB, 2), + (SECTOR_SIZE_8_MIB, 2), + (SECTOR_SIZE_16_MIB, 2), + (SECTOR_SIZE_512_MIB, 2), + (SECTOR_SIZE_1_GIB, 2), + (SECTOR_SIZE_32_GIB, 176), + (SECTOR_SIZE_64_GIB, 176), + ] + .iter() + .copied() + .collect(), + )) + } + + pub fn get_mut(&self) -> RwLockWriteGuard<'_, HashMap> { + self.0.write().expect("POREP_MINIMUM_CHALLENGES poisoned") + } + + pub fn from_sector_size(&self, sector_size: u64) -> usize { + match self + .0 + .read() + .expect("POREP_MINIMUM_CHALLENGES poisoned") + .get(§or_size) + { + Some(c) => *c, + None => panic!("invalid sector size"), + } + } +} + lazy_static! { - pub static ref POREP_MINIMUM_CHALLENGES: RwLock> = RwLock::new( - [ - (SECTOR_SIZE_2_KIB, 2), - (SECTOR_SIZE_4_KIB, 2), - (SECTOR_SIZE_16_KIB, 2), - (SECTOR_SIZE_32_KIB, 2), - (SECTOR_SIZE_8_MIB, 2), - (SECTOR_SIZE_16_MIB, 2), - (SECTOR_SIZE_512_MIB, 2), - (SECTOR_SIZE_1_GIB, 2), - (SECTOR_SIZE_32_GIB, 176), - (SECTOR_SIZE_64_GIB, 176), - ] - .iter() - .copied() - .collect() - ); + pub static ref POREP_MINIMUM_CHALLENGES: PorepMinimumChallenges = PorepMinimumChallenges::new(); pub static ref POREP_PARTITIONS: RwLock> = RwLock::new( [ (SECTOR_SIZE_2_KIB, 1), @@ -140,12 +163,17 @@ pub type DefaultPieceDomain = as Hasher>::Domain; /// The default hasher for merkle trees currently in use. pub type DefaultTreeHasher = PoseidonHasher; pub type DefaultTreeDomain = as Hasher>::Domain; - pub type DefaultOctTreeStore = storage_proofs_core::merkle::LCStore>; +/// A binary merkle tree with SHA256 hashing, where all levels have arity 2. It's fully +/// persisted to disk. pub type DefaultBinaryTree = BinaryMerkleTree>; -pub type DefaultOctTree = OctMerkleTree>; -pub type DefaultOctLCTree = OctLCMerkleTree>; +/// A merkle tree with Poseidon hashing, where all levels have arity 8. It's fully persisted to +/// disk. +pub type DefaultOctTree = DiskTree, U8, U0, U0>; +/// A merkle tree with Poseidon hashing, where all levels have arity 8. Some levels are not +/// persisted to disk, but only cached in memory. +pub type DefaultOctLCTree = LCTree, U8, U0, U0>; // Generic shapes pub type SectorShapeBase = LCTree, U8, U0, U0>; diff --git a/filecoin-proofs/src/parameters.rs b/filecoin-proofs/src/parameters.rs index a8da81817..7db393ffc 100644 --- a/filecoin-proofs/src/parameters.rs +++ b/filecoin-proofs/src/parameters.rs @@ -106,11 +106,7 @@ pub fn setup_params( let layer_challenges = if is_groth16_field::() { select_challenges( partitions, - *POREP_MINIMUM_CHALLENGES - .read() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .get(§or_bytes) - .expect("unknown sector size") as usize, + POREP_MINIMUM_CHALLENGES.from_sector_size(sector_bytes), num_layers, ) } else { diff --git a/filecoin-proofs/src/types/porep_config.rs b/filecoin-proofs/src/types/porep_config.rs index b923cb869..ac390b68c 100644 --- a/filecoin-proofs/src/types/porep_config.rs +++ b/filecoin-proofs/src/types/porep_config.rs @@ -17,6 +17,7 @@ use crate::{ constants::DefaultPieceHasher, parameters::public_params, types::{PaddedBytesAmount, PoRepProofPartitions, SectorSize, UnpaddedBytesAmount}, + POREP_PARTITIONS, }; #[derive(Clone, Copy, Debug)] @@ -56,6 +57,7 @@ impl From for SectorSize { } impl PoRepConfig { + /// construct PoRepConfig by halo2 pub fn new_halo2(sector_size: SectorSize, porep_id: [u8; 32], api_version: ApiVersion) -> Self { let sector_nodes = u64::from(sector_size) as usize >> 5; let partitions = storage_proofs_porep::stacked::halo2::partition_count(sector_nodes); @@ -67,6 +69,22 @@ impl PoRepConfig { } } + /// construct PoRepConfig by groth16 + pub fn new_groth16(sector_size: u64, porep_id: [u8; 32], api_version: ApiVersion) -> Self { + Self { + sector_size: SectorSize(sector_size), + partitions: PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(§or_size) + .expect("unknown sector size") as usize, + ), + porep_id, + api_version, + } + } + /// Returns the cache identifier as used by `storage-proofs::parameter_cache`. pub fn get_cache_identifier(&self) -> Result where diff --git a/filecoin-proofs/tests/mod.rs b/filecoin-proofs/tests/mod.rs index 9b07cc73a..c23c8e1e1 100644 --- a/filecoin-proofs/tests/mod.rs +++ b/filecoin-proofs/tests/mod.rs @@ -4,7 +4,7 @@ use blstrs::Scalar as Fr; use ff::Field; use filecoin_proofs::{ as_safe_commitment, verify_seal, DefaultOctLCTree, DefaultTreeDomain, PoRepConfig, - PoRepProofPartitions, SectorSize, POREP_PARTITIONS, SECTOR_SIZE_2_KIB, TEST_SEED, + SECTOR_SIZE_2_KIB, TEST_SEED, }; use fr32::bytes_into_fr; use rand::SeedableRng; @@ -26,18 +26,7 @@ fn test_verify_seal_fr32_validation() { // Test failure for invalid comm_r conversion. { let result = verify_seal::>( - PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_2_KIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_2_KIB) - .expect("unknown sector size") as usize, - ), - porep_id: arbitrary_porep_id, - api_version: ApiVersion::V1_1_0, - }, + PoRepConfig::new_groth16(SECTOR_SIZE_2_KIB, arbitrary_porep_id, ApiVersion::V1_1_0), not_convertible_to_fr_bytes, convertible_to_fr_bytes, [0; 32], @@ -65,18 +54,7 @@ fn test_verify_seal_fr32_validation() { // Test failure for invalid comm_d conversion. { let result = verify_seal::>( - PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_2_KIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_2_KIB) - .expect("unknown sector size") as usize, - ), - porep_id: arbitrary_porep_id, - api_version: ApiVersion::V1_1_0, - }, + PoRepConfig::new_groth16(SECTOR_SIZE_2_KIB, arbitrary_porep_id, ApiVersion::V1_1_0), convertible_to_fr_bytes, not_convertible_to_fr_bytes, [0; 32], @@ -108,18 +86,7 @@ fn test_verify_seal_fr32_validation() { assert!(out.is_ok(), "tripwire"); let result = verify_seal::>( - PoRepConfig { - sector_size: SectorSize(SECTOR_SIZE_2_KIB), - partitions: PoRepProofPartitions( - *POREP_PARTITIONS - .read() - .expect("POREP_PARTITIONS poisoned") - .get(&SECTOR_SIZE_2_KIB) - .expect("unknown sector size") as usize, - ), - porep_id: arbitrary_porep_id, - api_version: ApiVersion::V1_1_0, - }, + PoRepConfig::new_groth16(SECTOR_SIZE_2_KIB, arbitrary_porep_id, ApiVersion::V1_1_0), non_zero_commitment_fr_bytes, non_zero_commitment_fr_bytes, [0; 32], diff --git a/filecoin-proofs/tests/upgrade.rs b/filecoin-proofs/tests/upgrade.rs index 63348bc9d..f3b85de53 100644 --- a/filecoin-proofs/tests/upgrade.rs +++ b/filecoin-proofs/tests/upgrade.rs @@ -28,7 +28,7 @@ use filecoin_proofs::{ SECTOR_SIZE_64_GIB, }; use log::info; -use memmap::MmapOptions; +use memmap2::MmapOptions; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; #[cfg(not(feature = "big-tests"))] diff --git a/fr32/Cargo.toml b/fr32/Cargo.toml index 884b34642..ac9f2c1d7 100644 --- a/fr32/Cargo.toml +++ b/fr32/Cargo.toml @@ -13,7 +13,7 @@ byte-slice-cast = "1.0.0" byteorder = "1" ff = "0.12.0" thiserror = "1.0.6" -blstrs = "0.5.0" +blstrs = "0.6.1" [dev-dependencies] bitvec = "0.17" diff --git a/storage-proofs-core/Cargo.toml b/storage-proofs-core/Cargo.toml index 7f7911b6c..676b2e0be 100644 --- a/storage-proofs-core/Cargo.toml +++ b/storage-proofs-core/Cargo.toml @@ -19,7 +19,7 @@ byteorder = "1" config = { version = "0.12.0", default-features = false, features = ["toml"] } itertools = "0.10.3" lazy_static = "1.2" -memmap = "0.7" +memmap2 = "0.5.6" aes = "0.8.1" sha2 = "0.10.2" fs2 = "0.4" @@ -27,7 +27,7 @@ rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} blake2b_simd = "1.0.0" ff = "0.12.0" -bellperson = "0.22.0" +bellperson = "0.24.0" serde_json = "1.0" log = "0.4.7" rand_chacha = "0.3" @@ -39,11 +39,12 @@ gperftools = { version = "0.2", optional = true } num_cpus = "1.10.1" semver = "1.0.6" fr32 = { path = "../fr32", version = "~5.0.0"} + +blstrs = "0.6.1" cbc = { version = "0.1.2", features = ["std"] } -blstrs = "0.5.0" halo2_proofs = "0.2.0" fil-halo2-gadgets = { path = "../fil-halo2-gadgets" } -ec-gpu = "0.1.0" +ec-gpu = "0.2.0" [dev-dependencies] proptest = "1.0.0" diff --git a/storage-proofs-core/benches/merkle_groth16_halo2.rs b/storage-proofs-core/benches/merkle_groth16_halo2.rs index e2d10bca3..5a44e40ea 100644 --- a/storage-proofs-core/benches/merkle_groth16_halo2.rs +++ b/storage-proofs-core/benches/merkle_groth16_halo2.rs @@ -20,8 +20,9 @@ use halo2_proofs::{ plonk::{self, Circuit, Column, ConstraintSystem, Instance}, }; use rand::thread_rng; +use storage_proofs_core::compound_proof::CompoundProof; +use storage_proofs_core::merkle::MerkleProofTrait; use storage_proofs_core::{ - compound_proof::CompoundProof as _, gadgets::por::{PoRCircuit, PoRCompound}, halo2::{ self, @@ -31,7 +32,7 @@ use storage_proofs_core::{ }, CircuitRows, Halo2Field, Halo2Keypair, }, - merkle::{MerkleProofTrait, MerkleTree, MerkleTreeTrait}, + merkle::{MerkleTree, MerkleTreeTrait}, por as vanilla, }; diff --git a/storage-proofs-core/src/data.rs b/storage-proofs-core/src/data.rs index 323d5dffc..47941ae38 100644 --- a/storage-proofs-core/src/data.rs +++ b/storage-proofs-core/src/data.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use anyhow::{ensure, Context, Result}; use log::info; -use memmap::{MmapMut, MmapOptions}; +use memmap2::{MmapMut, MmapOptions}; /// A wrapper around data either on disk or a slice in memory, that can be dropped and read back into memory, /// to allow for better control of memory consumption. diff --git a/storage-proofs-core/src/drgraph.rs b/storage-proofs-core/src/drgraph.rs index 755bc202d..064a60224 100644 --- a/storage-proofs-core/src/drgraph.rs +++ b/storage-proofs-core/src/drgraph.rs @@ -264,7 +264,7 @@ mod tests { }; use generic_array::typenum::{U0, U2, U4, U8}; use halo2_proofs::pasta::{Fp, Fq}; - use memmap::{MmapMut, MmapOptions}; + use memmap2::{MmapMut, MmapOptions}; use merkletree::store::StoreConfig; use crate::merkle::{ diff --git a/storage-proofs-core/src/merkle/builders.rs b/storage-proofs-core/src/merkle/builders.rs index 9f75669fb..39b37f6d4 100644 --- a/storage-proofs-core/src/merkle/builders.rs +++ b/storage-proofs-core/src/merkle/builders.rs @@ -19,10 +19,14 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use crate::{ error::{Error, Result}, - merkle::{DiskTree, LCMerkleTree, LCStore, LCTree, MerkleTreeTrait, MerkleTreeWrapper}, + merkle::{DiskTree, LCTree, MerkleTreeTrait, MerkleTreeWrapper}, util::{data_at_node, default_rows_to_discard, NODE_SIZE}, }; +/// A tree where the specified arity is used for all the levels. Some levels are not persisted to +/// disk, but only cached in memory. +type LCMerkleTree = LCTree; + // Create a DiskTree from the provided config(s), each representing a 'base' layer tree with 'base_tree_len' elements. pub fn create_disk_tree( base_tree_len: usize, @@ -76,7 +80,7 @@ pub fn create_lc_tree( LCTree::from_store_configs_and_replica(base_tree_leafs, configs, replica_config) } else { ensure!(configs.len() == 1, "Invalid tree-shape specified"); - let store = LCStore::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, Tree::Arity::to_usize(), &configs[0], diff --git a/storage-proofs-core/src/merkle/mod.rs b/storage-proofs-core/src/merkle/mod.rs index ca196849c..7526647f0 100644 --- a/storage-proofs-core/src/merkle/mod.rs +++ b/storage-proofs-core/src/merkle/mod.rs @@ -5,7 +5,7 @@ use std::fs::File; pub use merkletree::store::{DiskStore, ExternalReader, Store}; use filecoin_hashers::Hasher; -use generic_array::typenum::{U0, U2, U4, U8}; +use generic_array::typenum::{U0, U2}; use merkletree::store::LevelCacheStore; mod builders; @@ -18,26 +18,30 @@ pub use tree::*; pub type LCStore = LevelCacheStore; -pub type MerkleStore = DiskStore; - -pub type DiskTree = MerkleTreeWrapper::Domain>, U, V, W>; -pub type LCTree = MerkleTreeWrapper::Domain>, U, V, W>; - pub type MerkleTree = DiskTree; -pub type LCMerkleTree = LCTree; - -pub type BinaryMerkleTree = MerkleTree; -pub type BinaryLCMerkleTree = LCMerkleTree; -pub type BinarySubMerkleTree = DiskTree; - -pub type QuadMerkleTree = MerkleTree; -pub type QuadLCMerkleTree = LCMerkleTree; - -pub type OctMerkleTree = DiskTree; -pub type OctSubMerkleTree = DiskTree; -pub type OctTopMerkleTree = DiskTree; +/// A tree that is fully persisted to disk. +/// +/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top-tree +/// arity `W`. +/// +/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or +/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would +/// create a tree where the top level has two children, the levels below 4 children and all other +/// levels below have 8 children. +pub type DiskTree = MerkleTreeWrapper::Domain>, U, V, W>; -pub type OctLCMerkleTree = LCTree; -pub type OctLCSubMerkleTree = LCTree; -pub type OctLCTopMerkleTree = LCTree; +/// A tree that is partially stored on disk, some levels are in memory. +/// +/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree +/// arity `W`. +/// +/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or +/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would +/// create a tree where the top level has two children, the levels below 4 children and all other +/// levels below have 8 children. +pub type LCTree = + MerkleTreeWrapper::Domain, File>, U, V, W>; + +/// A binary merkle tree, where all levels have arity 2. It's fully persisted to disk. +pub type BinaryMerkleTree = DiskTree; diff --git a/storage-proofs-core/src/merkle/tree.rs b/storage-proofs-core/src/merkle/tree.rs index 508a9c70a..d38fc8124 100644 --- a/storage-proofs-core/src/merkle/tree.rs +++ b/storage-proofs-core/src/merkle/tree.rs @@ -5,7 +5,6 @@ use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use anyhow::Result; -use ec_gpu::GpuField; use ff::PrimeField; use filecoin_hashers::{Hasher, PoseidonArity}; use generic_array::typenum::U0; @@ -23,7 +22,10 @@ pub trait MerkleTreeTrait: Send + Sync + Debug { type Arity: PoseidonArity; type SubTreeArity: PoseidonArity; type TopTreeArity: PoseidonArity; - type Field: PrimeField + GpuField; + #[cfg(not(any(feature = "cuda", feature = "opencl")))] + type Field: PrimeField; + #[cfg(any(feature = "cuda", feature = "opencl"))] + type Field: PrimeField + ec_gpu::GpuName; type Hasher: 'static + Hasher; type Store: Store<::Domain>; type Proof: MerkleProofTrait< diff --git a/storage-proofs-core/src/parameter_cache.rs b/storage-proofs-core/src/parameter_cache.rs index b68088d4c..48a7b788c 100644 --- a/storage-proofs-core/src/parameter_cache.rs +++ b/storage-proofs-core/src/parameter_cache.rs @@ -13,7 +13,7 @@ use fs2::FileExt; use itertools::Itertools; use lazy_static::lazy_static; use log::info; -use memmap::MmapOptions; +use memmap2::MmapOptions; use rand::RngCore; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; diff --git a/storage-proofs-core/src/test_helper.rs b/storage-proofs-core/src/test_helper.rs index 83fa85665..d4b3abbc6 100644 --- a/storage-proofs-core/src/test_helper.rs +++ b/storage-proofs-core/src/test_helper.rs @@ -2,7 +2,7 @@ use std::fs::OpenOptions; use std::io::Write; use std::path::Path; -use memmap::{MmapMut, MmapOptions}; +use memmap2::{MmapMut, MmapOptions}; pub fn setup_replica(data: &[u8], replica_path: &Path) -> MmapMut { let mut f = OpenOptions::new() diff --git a/storage-proofs-porep/Cargo.toml b/storage-proofs-porep/Cargo.toml index 68087b63e..21f360b46 100644 --- a/storage-proofs-porep/Cargo.toml +++ b/storage-proofs-porep/Cargo.toml @@ -22,13 +22,12 @@ rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" ff = "0.12.0" -bellperson = "0.22.0" +bellperson = "0.24.0" log = "0.4.7" pretty_assertions = "1.2.0" generic-array = "0.14.4" anyhow = "1.0.23" -# neptune = { version = "~7.0.0", features = ["bls", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } -neptune = { git = "https://github.com/filecoin-project/neptune", branch = "halo2", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { version = "~8.1.0", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } num_cpus = "1.10.1" hex = "0.4.2" bincode = "1.1.2" @@ -40,7 +39,7 @@ libc = "0.2" fdlimit = "0.2.0" fr32 = { path = "../fr32", version = "~5.0.0", default-features = false } yastl = "0.1.2" -blstrs = "0.5.0" +blstrs = "0.6.1" typemap = "0.3.3" halo2_proofs = "0.2.0" fil-halo2-gadgets = { path = "../fil-halo2-gadgets" } diff --git a/storage-proofs-porep/src/drg/vanilla.rs b/storage-proofs-porep/src/drg/vanilla.rs index 5e0a5257a..fc5ebd3a7 100644 --- a/storage-proofs-porep/src/drg/vanilla.rs +++ b/storage-proofs-porep/src/drg/vanilla.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use anyhow::{ensure, Context}; use filecoin_hashers::{Domain, HashFunction, Hasher, PoseidonArity}; use fr32::bytes_into_fr_repr_safe; -use generic_array::typenum::U2; +use generic_array::typenum::{U0, U2}; use merkletree::store::{ReplicaConfig, StoreConfig}; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; @@ -16,8 +16,8 @@ use storage_proofs_core::{ drgraph::Graph, error::Result, merkle::{ - create_base_lcmerkle_tree, create_base_merkle_tree, BinaryLCMerkleTree, BinaryMerkleTree, - LCMerkleTree, MerkleProof, MerkleProofTrait, MerkleTreeTrait, + create_base_lcmerkle_tree, create_base_merkle_tree, BinaryMerkleTree, LCTree, MerkleProof, + MerkleProofTrait, MerkleTreeTrait, }, parameter_cache::ParameterSetMetadata, proof::{NoRequirements, ProofScheme}, @@ -27,6 +27,10 @@ use storage_proofs_core::{ use crate::{encode, PoRep}; +/// A binary merkle tree, where all levels have arity 2. Some levels are not persisted to disk, but +/// only cached in memory. +type BinaryLCMerkleTree = LCTree; + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct Tau { pub comm_r: T, @@ -580,25 +584,23 @@ pub fn decode_domain_block( node: usize, node_data: H::Domain, parents: &[u32], -) -> Result { - let key = create_key_from_tree::(replica_id, node, parents, tree)?; +) -> Result +where + H: Hasher, +{ + let key = create_key_from_tree::(replica_id, node, parents, tree)?; Ok(encode::decode(key, node_data)) } /// Creates the encoding key from a `MerkleTree`. -/// The algorithm for that is `Blake2s(id | encodedParentNode1 | encodedParentNode1 | ...)`. -/// It is only public so that it can be used for benchmarking -pub fn create_key_from_tree( +/// The algorithm for that is `Poseidon(id | encodedParentNode1 | encodedParentNode1 | ...)`. +fn create_key_from_tree( id: &H::Domain, node: usize, parents: &[u32], - tree: &LCMerkleTree, -) -> Result -where - H: Hasher, - U: PoseidonArity, -{ + tree: &BinaryLCMerkleTree, +) -> Result { let mut hasher = Sha256::new(); hasher.update(AsRef::<[u8]>::as_ref(&id)); diff --git a/storage-proofs-porep/src/stacked/halo2/circuit.rs b/storage-proofs-porep/src/stacked/halo2/circuit.rs index 063712d2f..932ca5099 100644 --- a/storage-proofs-porep/src/stacked/halo2/circuit.rs +++ b/storage-proofs-porep/src/stacked/halo2/circuit.rs @@ -1348,8 +1348,7 @@ where PoseidonHasher: Hasher, { pub fn proof_size(&self, batch_size: Option) -> usize { - let k = self.k() as usize; - let cost = halo2_proofs::dev::CircuitCost::::measure(k, self); + let cost = halo2_proofs::dev::CircuitCost::::measure(self.k(), self); match batch_size { Some(batch_size) => cost.proof_size(batch_size).into(), None => cost.marginal_proof_size().into(), diff --git a/storage-proofs-porep/src/stacked/vanilla/cores.rs b/storage-proofs-porep/src/stacked/vanilla/cores.rs index 850b03a74..c7b6632f5 100644 --- a/storage-proofs-porep/src/stacked/vanilla/cores.rs +++ b/storage-proofs-porep/src/stacked/vanilla/cores.rs @@ -157,6 +157,7 @@ fn create_core_units( core_count: usize, group_count: usize, cores_per_unit: usize, + allowed_cores: &hwloc::CpuSet, ) -> Vec> { assert_eq!(0, core_count % group_count); // The number of cores that belong to a single group. @@ -176,7 +177,7 @@ fn create_core_units( let core_units = (0..unit_count) .map(|i| { (0..cores_per_unit) - .map(|j| { + .filter_map(|j| { // Every group gets a single unit assigned first. Only if all groups have // already one unit, a second one will be assigned if possible. This would then // be the second "round" of assignments. @@ -184,10 +185,14 @@ fn create_core_units( // The index of the core that is bound to a unit. let core_index = (j + i * group_size) % core_count + (round * cores_per_unit); assert!(core_index < core_count); - core_index + + allowed_cores + .is_set(core_index.try_into().ok()?) + .then_some(core_index) }) .collect::>() }) + .filter(|x| !x.is_empty()) .collect::>(); debug!("Core units: {:?}", core_units); core_units @@ -229,6 +234,15 @@ fn core_units(cores_per_unit: usize) -> Option>> { let all_cores = topo .objects_with_type(&ObjectType::Core) .expect("objects_with_type failed"); + + let allowed_cores = topo + .get_cpubind(hwloc::CpuBindFlags::empty()) + .unwrap_or_else(|| { + topo.object_at_root() + .allowed_cpuset() + .unwrap_or_else(hwloc::CpuSet::full) + }); + // The total number of physical cores, even across packages. let core_count = all_cores.len(); @@ -238,7 +252,8 @@ fn core_units(cores_per_unit: usize) -> Option>> { let group_count = get_shared_cache_count(&topo, core_depth, core_count); // The list of units the multicore SDR threads can be bound to. - let core_units = create_core_units(core_count, group_count, cores_per_unit); + let core_units = create_core_units(core_count, group_count, cores_per_unit, &allowed_cores); + // this needs to take the all_cores vec instead of just a core count Some( core_units .iter() @@ -283,13 +298,13 @@ mod tests { fn test_create_core_units() { fil_logger::maybe_init(); - let ci = create_core_units(18, 1, 4); + let ci = create_core_units(18, 1, 4, &(0..18).collect()); assert_eq!( ci, [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]] ); - let dc = create_core_units(32, 2, 4); + let dc = create_core_units(32, 2, 4, &(0..32).collect()); assert_eq!( dc, [ @@ -304,28 +319,28 @@ mod tests { ] ); - let amd = create_core_units(16, 4, 4); + let amd = create_core_units(16, 4, 4, &(0..16).collect()); assert_eq!( amd, [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]] ); - let amd_not_filled = create_core_units(16, 4, 3); + let amd_not_filled = create_core_units(16, 4, 3, &(0..16).collect()); assert_eq!( amd_not_filled, [[0, 1, 2], [4, 5, 6], [8, 9, 10], [12, 13, 14]] ); - let amd_not_filled = create_core_units(16, 4, 3); + let amd_not_filled = create_core_units(16, 4, 3, &(0..16).collect()); assert_eq!( amd_not_filled, [[0, 1, 2], [4, 5, 6], [8, 9, 10], [12, 13, 14]] ); - let intel = create_core_units(16, 2, 3); + let intel = create_core_units(16, 2, 3, &(0..16).collect()); assert_eq!(intel, [[0, 1, 2], [8, 9, 10], [3, 4, 5], [11, 12, 13]]); - let sp = create_core_units(48, 8, 3); + let sp = create_core_units(48, 8, 3, &(0..48).collect()); assert_eq!( sp, [ @@ -348,7 +363,7 @@ mod tests { ] ); - let sp_not_filled = create_core_units(48, 8, 4); + let sp_not_filled = create_core_units(48, 8, 4, &(0..48).collect()); assert_eq!( sp_not_filled, [ @@ -363,9 +378,73 @@ mod tests { ] ); - let laptop = create_core_units(4, 1, 2); + let laptop = create_core_units(4, 1, 2, &(0..4).collect()); assert_eq!(laptop, [[0, 1], [2, 3]]); - let laptop_not_filled = create_core_units(4, 1, 3); + let laptop_not_filled = create_core_units(4, 1, 3, &(0..4).collect()); assert_eq!(laptop_not_filled, [[0, 1, 2]]); + + let amd_limited_0 = create_core_units(16, 4, 4, &(0..8).collect()); + assert_eq!(amd_limited_0, [[0, 1, 2, 3], [4, 5, 6, 7]]); + + let amd_limited_1 = create_core_units(16, 4, 4, &(8..16).collect()); + assert_eq!(amd_limited_1, [[8, 9, 10, 11], [12, 13, 14, 15]]); + + let sp_limited_0 = create_core_units(48, 8, 3, &(0..24).collect()); + assert_eq!( + sp_limited_0, + [ + [0, 1, 2], + [6, 7, 8], + [12, 13, 14], + [18, 19, 20], + [3, 4, 5], + [9, 10, 11], + [15, 16, 17], + [21, 22, 23], + ] + ); + + let sp_limited_1 = create_core_units(48, 8, 3, &(24..48).collect()); + assert_eq!( + sp_limited_1, + [ + [24, 25, 26], + [30, 31, 32], + [36, 37, 38], + [42, 43, 44], + [27, 28, 29], + [33, 34, 35], + [39, 40, 41], + [45, 46, 47] + ] + ); + + let limited_group = create_core_units( + 16, + 4, + 4, + &vec![0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14] + .into_iter() + .collect(), + ); + assert_eq!( + limited_group, + [[0, 1, 2], [4, 5, 6], [8, 9, 10], [12, 13, 14],] + ); + + let limited_non_continuous = create_core_units(48, 8, 3, &(0..12).chain(24..36).collect()); + assert_eq!( + limited_non_continuous, + [ + [0, 1, 2], + [6, 7, 8], + [24, 25, 26], + [30, 31, 32], + [3, 4, 5], + [9, 10, 11], + [27, 28, 29], + [33, 34, 35], + ] + ); } } diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index aa0ed1d47..098a1934c 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -29,7 +29,7 @@ use storage_proofs_core::{ measurements::{measure_op, Operation}, merkle::{ create_disk_tree, create_lc_tree, get_base_tree_count, split_config, - split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait, MerkleTree, + split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait, MerkleTreeTrait, }, settings::SETTINGS, @@ -436,7 +436,7 @@ where let leafs = tree_data.len() / NODE_SIZE; assert_eq!(tree_data.len() % NODE_SIZE, 0); - let tree = MerkleTree::from_par_iter_with_config( + let tree = BinaryMerkleTree::from_par_iter_with_config( (0..leafs) .into_par_iter() // TODO: proper error handling instead of `unwrap()` diff --git a/storage-proofs-post/Cargo.toml b/storage-proofs-post/Cargo.toml index 9bbba5681..399c1e281 100644 --- a/storage-proofs-post/Cargo.toml +++ b/storage-proofs-post/Cargo.toml @@ -19,13 +19,13 @@ rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} blake2b_simd = "1.0.0" ff = "0.12.0" -bellperson = "0.22.0" +bellperson = "0.24.0" log = "0.4.7" hex = "0.4.0" generic-array = "0.14.4" anyhow = "1.0.23" fr32 = { path = "../fr32", version = "~5.0.0", default-features = false } -blstrs = "0.5.0" +blstrs = "0.6.1" halo2_proofs = "0.2.0" fil-halo2-gadgets = { path = "../fil-halo2-gadgets" } diff --git a/storage-proofs-post/tests/fallback_circuit.rs b/storage-proofs-post/tests/fallback_circuit.rs index f65766d97..c278c95cf 100644 --- a/storage-proofs-post/tests/fallback_circuit.rs +++ b/storage-proofs-post/tests/fallback_circuit.rs @@ -12,7 +12,7 @@ use storage_proofs_core::{ api_version::ApiVersion, compound_proof::CompoundProof, error::Result, - merkle::{generate_tree, get_base_tree_count, LCTree, MerkleTreeTrait, OctMerkleTree}, + merkle::{generate_tree, get_base_tree_count, DiskTree, LCTree, MerkleTreeTrait}, proof::ProofScheme, util::NODE_SIZE, TEST_SEED, @@ -212,11 +212,11 @@ fn test_fallback_post_circuit_poseidon_base_8_bench_cs() { api_version: ApiVersion::V1_1_0, }; - let pp = FallbackPoSt::>>::setup(¶ms) + let pp = FallbackPoSt::, U8, U0, U0>>::setup(¶ms) .expect("fallback post setup failure"); let mut cs = BenchCS::::new(); - FallbackPoStCompound::>>::blank_circuit(&pp) + FallbackPoStCompound::, U8, U0, U0>>::blank_circuit(&pp) .synthesize(&mut cs) .expect("blank circuit failure"); diff --git a/storage-proofs-update/Cargo.toml b/storage-proofs-update/Cargo.toml index a78f64073..abce494f8 100644 --- a/storage-proofs-update/Cargo.toml +++ b/storage-proofs-update/Cargo.toml @@ -17,17 +17,16 @@ merkletree = "0.22.0" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} ff = "0.12.0" -bellperson = "0.22.0" -blstrs = "0.5.0" +bellperson = "0.24.0" +blstrs = "0.6.1" log = "0.4.7" generic-array = "0.14.4" anyhow = "1.0.23" -# neptune = { version = "~7.0.0", features = ["bls", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } -neptune = { git = "https://github.com/filecoin-project/neptune", branch = "halo2", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { version = "~8.1.0", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } lazy_static = "1.2" fr32 = { path = "../fr32", version = "~5.0.0", default-features = false } fil_logger = "0.1.6" -memmap = "0.7" +memmap2 = "0.5.6" halo2_proofs = "0.2.0" typemap = "0.3.3" fil-halo2-gadgets = { path = "../fil-halo2-gadgets" } diff --git a/storage-proofs-update/src/constants.rs b/storage-proofs-update/src/constants.rs index 221cd64da..fb1dee633 100644 --- a/storage-proofs-update/src/constants.rs +++ b/storage-proofs-update/src/constants.rs @@ -19,6 +19,9 @@ use storage_proofs_core::{ }; use typemap::ShareMap; +// Use a custom domain separation tag when generating randomness phi, rho, and challenges bits. +pub const HASH_TYPE_GEN_RANDOMNESS: HashType = HashType::Custom(CType::Arbitrary(1)); + lazy_static! { // Use a custom domain separation tag `HashType` when using Poseidon to generate randomness // (i.e. phi, rho, and challenges bits). diff --git a/storage-proofs-update/src/vanilla.rs b/storage-proofs-update/src/vanilla.rs index 47146404f..1d26da9d8 100644 --- a/storage-proofs-update/src/vanilla.rs +++ b/storage-proofs-update/src/vanilla.rs @@ -9,7 +9,7 @@ use ff::PrimeField; use filecoin_hashers::{Domain, HashFunction, Hasher, PoseidonArity, PoseidonLookup}; use generic_array::typenum::{Unsigned, U2}; use log::{info, trace}; -use memmap::{Mmap, MmapMut, MmapOptions}; +use memmap2::{Mmap, MmapMut, MmapOptions}; use merkletree::{ merkle::{get_merkle_tree_leafs, get_merkle_tree_len}, store::{DiskStore, Store, StoreConfig},