From a9a47a4cd676de4c176f6d169d25ec6369aa9ae6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 7 Dec 2021 00:02:13 +0100 Subject: [PATCH] feat(solc): add support for compiling solc in parallel (#652) * docs: more docs and tracing * feat: add compile many * feat: add compile many * fix: make fields optional * chore: add num_cpus and criterion * add compile benchmark * feat: add job option * feat: add parallel compilation support * use ful utilization * chore: move pathmap to cache * fix: async write all * chore: clean up --- Cargo.lock | 227 +++++++++++++++++++ ethers-solc/Cargo.toml | 6 + ethers-solc/benches/compile_many.rs | 55 +++++ ethers-solc/src/artifacts.rs | 22 +- ethers-solc/src/cache.rs | 62 +++++- ethers-solc/src/compile.rs | 86 +++++++- ethers-solc/src/lib.rs | 324 ++++++++++++++++++---------- 7 files changed, 663 insertions(+), 119 deletions(-) create mode 100644 ethers-solc/benches/compile_many.rs diff --git a/Cargo.lock b/Cargo.lock index 150197b09..2f35fb75f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,6 +318,18 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.8.0" @@ -382,6 +394,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", +] + [[package]] name = "cc" version = "1.0.72" @@ -643,6 +664,88 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "futures", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -694,6 +797,28 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.7.0" @@ -1187,6 +1312,7 @@ name = "ethers-solc" version = "0.1.0" dependencies = [ "colored", + "criterion", "ethers-core", "futures-util", "getrandom 0.2.3", @@ -1194,6 +1320,7 @@ dependencies = [ "hex", "home", "md-5", + "num_cpus", "once_cell", "regex", "semver", @@ -1476,6 +1603,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "harp" version = "0.1.0" @@ -1829,6 +1962,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memory_units" version = "0.4.0" @@ -1963,6 +2105,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -2168,6 +2316,34 @@ version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.15" @@ -2360,6 +2536,31 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -2399,6 +2600,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.25" @@ -2798,6 +3005,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.130" @@ -3182,6 +3399,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.5.1" diff --git a/ethers-solc/Cargo.toml b/ethers-solc/Cargo.toml index fbd7a5927..9dfa8fb40 100644 --- a/ethers-solc/Cargo.toml +++ b/ethers-solc/Cargo.toml @@ -30,6 +30,7 @@ colored = "2.0.0" svm = { package = "svm-rs", version = "0.2.0", optional = true } glob = "0.3.0" tracing = "0.1.29" +num_cpus = "1.13.0" [target.'cfg(not(any(target_arch = "x86", target_arch = "x86_64")))'.dependencies] sha2 = { version = "0.9.8", default-features = false } @@ -45,9 +46,14 @@ home = "0.5.3" getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] +criterion = { version = "0.3", features = ["async_tokio"] } tokio = { version = "1.12.0", features = ["full"] } tempdir = "0.3.7" +[[bench]] +name = "compile_many" +harness = false + [features] async = ["tokio", "futures-util"] full = ["async", "svm"] diff --git a/ethers-solc/benches/compile_many.rs b/ethers-solc/benches/compile_many.rs new file mode 100644 index 000000000..ee2e9f5ad --- /dev/null +++ b/ethers-solc/benches/compile_many.rs @@ -0,0 +1,55 @@ +//! compile many benches +#[macro_use] +extern crate criterion; + +use criterion::Criterion; +use ethers_solc::{CompilerInput, Solc}; +use std::path::Path; + +fn compile_many_benchmark(c: &mut Criterion) { + let inputs = load_compiler_inputs(); + let solc = Solc::default(); + + let mut group = c.benchmark_group("compile many"); + group.sample_size(10); + group.bench_function("sequential", |b| { + b.iter(|| { + for i in inputs.iter() { + let _ = solc.compile(i).unwrap(); + } + }); + }); + + #[cfg(feature = "full")] + { + let tasks = inputs.into_iter().map(|input| (Solc::default(), input)).collect::>(); + let num = tasks.len(); + group.bench_function("concurrently", |b| { + b.to_async(tokio::runtime::Runtime::new().unwrap()).iter(|| async { + let _ = Solc::compile_many(tasks.clone(), num).await.flattened().unwrap(); + }); + }); + } +} + +fn load_compiler_inputs() -> Vec { + let mut inputs = Vec::new(); + for file in std::fs::read_dir(Path::new(&env!("CARGO_MANIFEST_DIR")).join("test-data/in")) + .unwrap() + .into_iter() + .take(5) + { + let file = file.unwrap(); + if file.path().to_string_lossy().as_ref().ends_with("20.json") { + // TODO needs support for parsing library placeholders first + continue + } + let input = std::fs::read_to_string(file.path()).unwrap(); + let input: CompilerInput = serde_json::from_str(&input).unwrap(); + inputs.push(input); + } + inputs +} + +criterion_group!(benches, compile_many_benchmark); +criterion_main!(benches); diff --git a/ethers-solc/src/artifacts.rs b/ethers-solc/src/artifacts.rs index 63ab7ec7a..075cf01ce 100644 --- a/ethers-solc/src/artifacts.rs +++ b/ethers-solc/src/artifacts.rs @@ -17,6 +17,8 @@ use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; /// An ordered list of files and their source pub type Sources = BTreeMap; +pub type Contracts = BTreeMap>; + /// Input type `solc` expects #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CompilerInput { @@ -453,7 +455,7 @@ pub struct CompilerOutput { #[serde(default)] pub sources: BTreeMap, #[serde(default)] - pub contracts: BTreeMap>, + pub contracts: Contracts, } impl CompilerOutput { @@ -592,7 +594,10 @@ impl CompactContract { impl From for CompactContract { fn from(c: Contract) -> Self { let (bin, bin_runtime) = if let Some(evm) = c.evm { - (Some(evm.bytecode.object), evm.deployed_bytecode.bytecode.map(|evm| evm.object)) + ( + Some(evm.bytecode.object), + evm.deployed_bytecode.and_then(|deployed| deployed.bytecode.map(|evm| evm.object)), + ) } else { (None, None) }; @@ -636,7 +641,9 @@ impl<'a> From<&'a Contract> for CompactContractRef<'a> { let (bin, bin_runtime) = if let Some(ref evm) = c.evm { ( Some(&evm.bytecode.object), - evm.deployed_bytecode.bytecode.as_ref().map(|evm| &evm.object), + evm.deployed_bytecode + .as_ref() + .and_then(|deployed| deployed.bytecode.as_ref().map(|evm| &evm.object)), ) } else { (None, None) @@ -684,7 +691,8 @@ pub struct Evm { #[serde(default, skip_serializing_if = "Option::is_none")] pub legacy_assembly: Option, pub bytecode: Bytecode, - pub deployed_bytecode: DeployedBytecode, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deployed_bytecode: Option, /// The list of function hashes #[serde(default, skip_serializing_if = "::std::collections::BTreeMap::is_empty")] pub method_identifiers: BTreeMap, @@ -703,9 +711,11 @@ pub struct Bytecode { #[serde(deserialize_with = "deserialize_bytes")] pub object: Bytes, /// Opcodes list (string) - pub opcodes: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub opcodes: Option, /// The source mapping as a string. See the source mapping definition. - pub source_map: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_map: Option, /// Array of sources generated by the compiler. Currently only contains a /// single Yul file. #[serde(default, skip_serializing_if = "Vec::is_empty")] diff --git a/ethers-solc/src/cache.rs b/ethers-solc/src/cache.rs index a13692861..4a5957e61 100644 --- a/ethers-solc/src/cache.rs +++ b/ethers-solc/src/cache.rs @@ -1,13 +1,13 @@ //! Support for compiling contracts use crate::{ - artifacts::Sources, + artifacts::{Contracts, Sources}, config::SolcConfig, error::{Result, SolcError}, utils, ArtifactOutput, }; use serde::{Deserialize, Serialize}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, fs::{self, File}, path::{Path, PathBuf}, time::{Duration, UNIX_EPOCH}, @@ -287,6 +287,64 @@ impl CacheEntry { } } +/// A helper type to handle source name/full disk mappings +/// +/// The disk path is the actual path where a file can be found on disk. +/// A source name is the internal identifier and is the remaining part of the disk path starting +/// with the configured source directory, (`contracts/contract.sol`) +#[derive(Debug, Default)] +pub struct PathMap { + /// all libraries to the source set while keeping track of their actual disk path + /// (`contracts/contract.sol` -> `/Users/.../contracts.sol`) + pub source_name_to_path: HashMap, + /// inverse of `source_name_to_path` : (`/Users/.../contracts.sol` -> `contracts/contract.sol`) + pub path_to_source_name: HashMap, + /* /// All paths, source names and actual file paths + * paths: Vec */ +} + +impl PathMap { + fn apply_mappings(sources: Sources, mappings: &HashMap) -> Sources { + sources + .into_iter() + .map(|(import, source)| { + if let Some(path) = mappings.get(&import).cloned() { + (path, source) + } else { + (import, source) + } + }) + .collect() + } + + /// Returns all contract names of the files mapped with the disk path + pub fn get_artifacts(&self, contracts: &Contracts) -> Vec<(PathBuf, Vec)> { + contracts + .iter() + .map(|(path, contracts)| { + let path = PathBuf::from(path); + let file = self.source_name_to_path.get(&path).cloned().unwrap_or(path); + (file, contracts.keys().cloned().collect::>()) + }) + .collect() + } + + pub fn extend(&mut self, other: PathMap) { + self.source_name_to_path.extend(other.source_name_to_path); + self.path_to_source_name.extend(other.path_to_source_name); + } + + /// Returns a new map with the source names as keys + pub fn set_source_names(&self, sources: Sources) -> Sources { + Self::apply_mappings(sources, &self.path_to_source_name) + } + + /// Returns a new map with the disk paths as keys + pub fn set_disk_paths(&self, sources: Sources) -> Sources { + Self::apply_mappings(sources, &self.source_name_to_path) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/ethers-solc/src/compile.rs b/ethers-solc/src/compile.rs index c7ebcbc7a..d52726da0 100644 --- a/ethers-solc/src/compile.rs +++ b/ethers-solc/src/compile.rs @@ -40,6 +40,7 @@ use once_cell::sync::Lazy; #[cfg(any(test, feature = "tests"))] use std::sync::Mutex; + #[cfg(any(test, feature = "tests"))] static LOCK: Lazy> = Lazy::new(|| Mutex::new(())); @@ -363,7 +364,7 @@ impl Solc { .stdout(Stdio::piped()) .spawn()?; let stdin = child.stdin.as_mut().unwrap(); - stdin.write(&content).await?; + stdin.write_all(&content).await?; stdin.flush().await?; compile_output(child.wait_with_output().await?) } @@ -380,6 +381,78 @@ impl Solc { .await?, ) } + + /// Compiles all `CompilerInput`s with their associated `Solc`. + /// + /// This will buffer up to `n` `solc` processes and then return the `CompilerOutput`s in the + /// order in which they complete. No more than `n` futures will be buffered at any point in + /// time, and less than `n` may also be buffered depending on the state of each future. + /// + /// # Example + /// + /// Compile 2 `CompilerInput`s at once + /// + /// ```no_run + /// # async fn example() { + /// use ethers_solc::{CompilerInput, Solc}; + /// let solc1 = Solc::default(); + /// let solc2 = Solc::default(); + /// let input1 = CompilerInput::new("contracts").unwrap(); + /// let input2 = CompilerInput::new("src").unwrap(); + /// + /// let outputs = Solc::compile_many([(solc1, input1), (solc2, input2)], 2).await.flattened().unwrap(); + /// # } + /// ``` + pub async fn compile_many(jobs: I, n: usize) -> CompiledMany + where + I: IntoIterator, + { + use futures_util::stream::StreamExt; + + let outputs = futures_util::stream::iter( + jobs.into_iter() + .map(|(solc, input)| async { (solc.async_compile(&input).await, solc, input) }), + ) + .buffer_unordered(n) + .collect::>() + .await; + CompiledMany { outputs } + } +} + +/// The result of a `solc` process bundled with its `Solc` and `CompilerInput` +type CompileElement = (Result, Solc, CompilerInput); + +/// The output of multiple `solc` processes. +#[derive(Debug)] +pub struct CompiledMany { + outputs: Vec, +} + +impl CompiledMany { + /// Returns an iterator over all output elements + pub fn outputs(&self) -> impl Iterator { + self.outputs.iter() + } + + /// Returns an iterator over all output elements + pub fn into_outputs(self) -> impl Iterator { + self.outputs.into_iter() + } + + /// Returns all `CompilerOutput` or the first error that occurred + pub fn flattened(self) -> Result> { + self.into_iter().collect() + } +} + +impl IntoIterator for CompiledMany { + type Item = Result; + type IntoIter = std::vec::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.outputs.into_iter().map(|(res, _, _)| res).collect::>().into_iter() + } } fn compile_output(output: Output) -> Result> { @@ -459,6 +532,17 @@ mod tests { let other = solc().async_compile(&serde_json::json!(input)).await.unwrap(); assert_eq!(out, other); } + #[cfg(feature = "async")] + #[tokio::test] + async fn async_solc_compile_works2() { + let input = include_str!("../test-data/in/compiler-in-2.json"); + let input: CompilerInput = serde_json::from_str(input).unwrap(); + let out = solc().async_compile(&input).await.unwrap(); + let other = solc().async_compile(&serde_json::json!(input)).await.unwrap(); + assert_eq!(out, other); + let sync_out = solc().compile(&input).unwrap(); + assert_eq!(out, sync_out); + } #[test] fn test_version_req() { diff --git a/ethers-solc/src/lib.rs b/ethers-solc/src/lib.rs index c47815f6d..5b23ea462 100644 --- a/ethers-solc/src/lib.rs +++ b/ethers-solc/src/lib.rs @@ -25,18 +25,14 @@ use crate::{artifacts::Source, cache::SolFilesCache}; pub mod error; pub mod utils; -use crate::artifacts::Sources; +use crate::{artifacts::Sources, cache::PathMap}; use error::Result; use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - convert::TryInto, - fmt, fs, io, - marker::PhantomData, + borrow::Cow, collections::BTreeMap, convert::TryInto, fmt, fs, io, marker::PhantomData, path::PathBuf, }; -/// Handles contract compiling +/// Represents a project workspace and handles `solc` compiling of all contracts in that workspace. #[derive(Debug)] pub struct Project { /// The layout of the @@ -57,6 +53,8 @@ pub struct Project { pub ignored_error_codes: Vec, /// The paths which will be allowed for library inclusion pub allowed_lib_paths: AllowedLibPaths, + /// Maximum number of `solc` processes to run simultaneously. + solc_jobs: usize, } impl Project { @@ -90,6 +88,12 @@ impl Project { } impl Project { + /// Sets the maximum number of parallel `solc` processes to run simultaneously. + pub fn set_solc_jobs(&mut self, jobs: usize) { + assert!(jobs > 0); + self.solc_jobs = jobs; + } + #[tracing::instrument(skip_all, name = "Project::write_cache_file")] fn write_cache_file( &self, @@ -121,10 +125,11 @@ impl Project { Ok(()) } - /// Returns all sources found under the project's sources path + /// Returns all sources found under the project's configured sources path #[tracing::instrument(skip_all, fields(name = "sources"))] pub fn sources(&self) -> io::Result { - Source::read_all_from(self.paths.sources.as_path()) + tracing::trace!("reading all sources from \"{}\"", self.paths.sources.display()); + Source::read_all_from(&self.paths.sources) } /// This emits the cargo [`rerun-if-changed`](https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorerun-if-changedpath) instruction. @@ -138,7 +143,7 @@ impl Project { /// /// ```no_run /// use ethers_solc::{Project, ProjectPathsConfig}; - /// // configure the project with all its paths, solc, cache etc. + /// // configure the project with all its paths, solc, cache etc. where the root dir is the current rust project. /// let project = Project::builder() /// .paths(ProjectPathsConfig::hardhat(env!("CARGO_MANIFEST_DIR")).unwrap()) /// .build() @@ -173,18 +178,15 @@ impl Project { /// /// NOTE: this does not check if the contracts were successfully compiled, see /// `CompilerOutput::has_error` instead. - /// NB: If the `svm` feature is enabled, this function will automatically detect /// solc versions across files. #[tracing::instrument(skip_all, name = "compile")] pub fn compile(&self) -> Result> { - tracing::trace!("sources"); let sources = self.sources()?; - tracing::trace!("done"); #[cfg(all(feature = "svm", feature = "async"))] if self.auto_detect { - tracing::trace!("auto-compile"); + tracing::trace!("using solc auto detection"); return self.svm_compile(sources) } @@ -199,7 +201,7 @@ impl Project { #[tracing::instrument(skip(self, sources))] fn svm_compile(&self, sources: Sources) -> Result> { use semver::{Version, VersionReq}; - use std::collections::hash_map; + use std::collections::hash_map::{self, HashMap}; // split them by version let mut sources_by_version = BTreeMap::new(); @@ -223,8 +225,8 @@ impl Project { version } }; - tracing::trace!("found installed solc \"{}\"", version); + // gets the solc binary for that version, it is expected tha this will succeed // AND find the solc since it was installed right above let mut solc = Solc::find_svm_installed_version(version.to_string())? @@ -239,29 +241,108 @@ impl Project { } tracing::trace!("preprocessing finished"); - let mut compiled = - ProjectCompileOutput::with_ignored_errors(self.ignored_error_codes.clone()); - - // run the compilation step for each version - tracing::trace!("compiling sources with viable solc versions"); - for (solc, sources) in sources_by_version { - let span = tracing::trace_span!("solc", "{}", solc.version_short()?); - let _enter = span.enter(); - + tracing::trace!("verifying solc checksums"); + for solc in sources_by_version.keys() { // verify that this solc version's checksum matches the checksum found remotely. If // not, re-install the same version. - let version = solc_versions.get(&solc.solc).unwrap(); - if let Err(_e) = solc.verify_checksum() { - tracing::trace!("corrupted solc version, redownloading..."); + let version = &solc_versions[&solc.solc]; + if solc.verify_checksum().is_err() { + tracing::trace!("corrupted solc version, redownloading \"{}\"", version); Solc::blocking_install(version)?; tracing::trace!("reinstalled solc: \"{}\"", version); } - // once matched, proceed to compile with it - tracing::trace!("compiling_with_version"); + } + + // run the compilation step for each version + let compiled = if self.solc_jobs > 1 && sources_by_version.len() > 1 { + self.compile_many(sources_by_version)? + } else { + self.compile_sources(sources_by_version)? + }; + tracing::trace!("compiled all sources"); + + Ok(compiled) + } + + #[cfg(all(feature = "svm", feature = "async"))] + fn compile_sources( + &self, + sources_by_version: BTreeMap>, + ) -> Result> { + tracing::trace!("compiling sources using a single solc job"); + let mut compiled = + ProjectCompileOutput::with_ignored_errors(self.ignored_error_codes.clone()); + for (solc, sources) in sources_by_version { + tracing::trace!( + "compiling {} sources with solc \"{}\"", + sources.len(), + solc.as_ref().display() + ); compiled.extend(self.compile_with_version(&solc, sources)?); - tracing::trace!("done compiling_with_version"); } - tracing::trace!("compiled sources with viable solc versions"); + Ok(compiled) + } + + #[cfg(all(feature = "svm", feature = "async"))] + fn compile_many( + &self, + sources_by_version: BTreeMap>, + ) -> Result> { + tracing::trace!("compiling sources using {} solc jobs", self.solc_jobs); + let mut compiled = + ProjectCompileOutput::with_ignored_errors(self.ignored_error_codes.clone()); + let mut paths = PathMap::default(); + let mut jobs = Vec::with_capacity(sources_by_version.len()); + + let mut all_sources = BTreeMap::default(); + let mut all_artifacts = Vec::with_capacity(sources_by_version.len()); + + // preprocess all sources + for (solc, sources) in sources_by_version { + match self.preprocess_sources(sources)? { + PreprocessedJob::Unchanged(artifacts) => { + compiled.extend(ProjectCompileOutput::from_unchanged(artifacts)); + } + PreprocessedJob::Items(sources, map, cached_artifacts) => { + compiled.extend_artifacts(cached_artifacts); + // replace absolute path with source name to make solc happy + let sources = map.set_source_names(sources); + paths.extend(map); + + let input = CompilerInput::with_sources(sources) + .normalize_evm_version(&solc.version()?) + .with_remappings(self.paths.remappings.clone()); + + jobs.push((solc, input)) + } + }; + } + + let outputs = tokio::runtime::Runtime::new() + .unwrap() + .block_on(Solc::compile_many(jobs, self.solc_jobs)); + + for (res, _, input) in outputs.into_outputs() { + let output = res?; + + if !output.has_error() { + if self.cached { + // get all contract names of the files and map them to the disk file + all_artifacts.extend(paths.get_artifacts(&output.contracts)); + all_sources.extend(paths.set_disk_paths(input.sources)); + } + + if !self.no_artifacts { + Artifacts::on_output(&output, &self.paths)?; + } + } + compiled.extend_output(output); + } + + // write the cache file + if self.cached { + self.write_cache_file(all_sources, all_artifacts)?; + } Ok(compiled) } @@ -276,23 +357,70 @@ impl Project { pub fn compile_with_version( &self, solc: &Solc, - mut sources: Sources, + sources: Sources, ) -> Result> { - let span = tracing::trace_span!("compiling"); - let _enter = span.enter(); - // add all libraries to the source set while keeping track of their actual disk path - // (`contracts/contract.sol` -> `/Users/.../contracts.sol`) - let mut source_name_to_path = HashMap::new(); - // inverse of `source_name_to_path` : (`/Users/.../contracts.sol` -> - // `contracts/contract.sol`) - let mut path_to_source_name = HashMap::new(); + let (sources, paths, cached_artifacts) = match self.preprocess_sources(sources)? { + PreprocessedJob::Unchanged(artifacts) => { + return Ok(ProjectCompileOutput::from_unchanged(artifacts)) + } + PreprocessedJob::Items(a, b, c) => (a, b, c), + }; + + tracing::trace!("compiling"); + + // replace absolute path with source name to make solc happy + let sources = paths.set_source_names(sources); + + let input = CompilerInput::with_sources(sources) + .normalize_evm_version(&solc.version()?) + .with_remappings(self.paths.remappings.clone()); + + tracing::trace!("calling solc with {} sources", input.sources.len()); + let output = solc.compile(&input)?; + tracing::trace!("compiled input, output has error: {}", output.has_error()); + + if output.has_error() { + return Ok(ProjectCompileOutput::from_compiler_output( + output, + self.ignored_error_codes.clone(), + )) + } + + if self.cached { + // get all contract names of the files and map them to the disk file + let artifacts = paths.get_artifacts(&output.contracts); + // reapply to disk paths + let sources = paths.set_disk_paths(input.sources); + // create cache file + self.write_cache_file(sources, artifacts)?; + } + + // TODO: There seems to be some type redundancy here, c.f. discussion with @mattsse + if !self.no_artifacts { + Artifacts::on_output(&output, &self.paths)?; + } + + Ok(ProjectCompileOutput::from_compiler_output_and_cache( + output, + cached_artifacts, + self.ignored_error_codes.clone(), + )) + } + + /// Preprocesses the given source files by resolving their libs and check against cache if + /// configured + fn preprocess_sources(&self, mut sources: Sources) -> Result> { + tracing::trace!("preprocessing sources files"); + + // keeps track of source names / disk paths + let mut paths = PathMap::default(); tracing::trace!("resolving libraries"); for (import, (source, path)) in self.resolved_libraries(&sources)? { // inserting with absolute path here and keep track of the source name <-> path mappings sources.insert(path.clone(), source); - path_to_source_name.insert(path.clone(), import.clone()); - source_name_to_path.insert(import, path); + paths.path_to_source_name.insert(path.clone(), import.clone()); + paths.source_name_to_path.insert(import, path); } tracing::trace!("resolved libraries"); @@ -321,60 +449,14 @@ impl Project { // if nothing changed and all artifacts still exist if changed_files.is_empty() { tracing::trace!("unchanged source files"); - return Ok(ProjectCompileOutput::from_unchanged(cached_artifacts)) + return Ok(PreprocessedJob::Unchanged(cached_artifacts)) } // There are changed files and maybe some cached files (changed_files, cached_artifacts) } else { (sources, BTreeMap::default()) }; - - // replace absolute path with source name to make solc happy - let sources = apply_mappings(sources, path_to_source_name); - - let input = CompilerInput::with_sources(sources) - .normalize_evm_version(&solc.version()?) - .with_remappings(self.paths.remappings.clone()); - tracing::trace!("calling solc with {} sources", input.sources.len()); - let output = solc.compile(&input)?; - tracing::trace!("compiled input, output has error: {}", output.has_error()); - - if output.has_error() { - return Ok(ProjectCompileOutput::from_compiler_output( - output, - self.ignored_error_codes.clone(), - )) - } - - if self.cached { - // get all contract names of the files and map them to the disk file - let artifacts = output - .contracts - .iter() - .map(|(path, contracts)| { - let path = PathBuf::from(path); - let file = source_name_to_path.get(&path).cloned().unwrap_or(path); - (file, contracts.keys().cloned().collect::>()) - }) - .collect::>(); - - // reapply to disk paths - let sources = apply_mappings(input.sources, source_name_to_path); - - // create cache file - self.write_cache_file(sources, artifacts)?; - } - - // TODO: There seems to be some type redundancy here, c.f. discussion with @mattsse - if !self.no_artifacts { - Artifacts::on_output(&output, &self.paths)?; - } - - Ok(ProjectCompileOutput::from_compiler_output_and_cache( - output, - cached_artifacts, - self.ignored_error_codes.clone(), - )) + Ok(PreprocessedJob::Items(sources, paths, cached_artifacts)) } /// Removes the project's artifacts and cache file @@ -392,17 +474,9 @@ impl Project { } } -fn apply_mappings(sources: Sources, mut mappings: HashMap) -> Sources { - sources - .into_iter() - .map(|(import, source)| { - if let Some(path) = mappings.remove(&import) { - (path, source) - } else { - (import, source) - } - }) - .collect() +enum PreprocessedJob { + Unchanged(BTreeMap), + Items(Sources, PathMap, BTreeMap), } pub struct ProjectBuilder { @@ -423,6 +497,7 @@ pub struct ProjectBuilder pub ignored_error_codes: Vec, /// All allowed paths pub allowed_paths: Vec, + solc_jobs: Option, } impl ProjectBuilder { @@ -464,6 +539,22 @@ impl ProjectBuilder { self } + /// Sets the maximum number of parallel `solc` processes to run simultaneously. + /// + /// # Panics + /// + /// `jobs` must be at least 1 + pub fn solc_jobs(mut self, jobs: usize) -> Self { + assert!(jobs > 0); + self.solc_jobs = Some(jobs); + self + } + + /// Sets the number of parallel `solc` processes to `1`, no parallelization + pub fn single_solc_jobs(self) -> Self { + self.solc_jobs(1) + } + /// Set arbitrary `ArtifactOutputHandler` pub fn artifacts(self) -> ProjectBuilder { let ProjectBuilder { @@ -475,6 +566,7 @@ impl ProjectBuilder { auto_detect, ignored_error_codes, allowed_paths, + solc_jobs, .. } = self; ProjectBuilder { @@ -487,6 +579,7 @@ impl ProjectBuilder { artifacts: PhantomData::default(), ignored_error_codes, allowed_paths, + solc_jobs, } } @@ -519,6 +612,7 @@ impl ProjectBuilder { artifacts, ignored_error_codes, mut allowed_paths, + solc_jobs, } = self; let solc = solc.unwrap_or_default(); @@ -541,6 +635,7 @@ impl ProjectBuilder { artifacts, ignored_error_codes, allowed_lib_paths: allowed_paths.try_into()?, + solc_jobs: solc_jobs.unwrap_or_else(::num_cpus::get), }) } } @@ -557,6 +652,7 @@ impl Default for ProjectBuilder { artifacts: PhantomData::default(), ignored_error_codes: Vec::new(), allowed_paths: vec![], + solc_jobs: None, } } } @@ -623,17 +719,25 @@ impl ProjectCompileOutput { pub fn extend(&mut self, compiled: ProjectCompileOutput) { let ProjectCompileOutput { compiler_output, artifacts, .. } = compiled; self.artifacts.extend(artifacts); - if let Some(compiled) = compiler_output { - if let Some(output) = self.compiler_output.as_mut() { - output.errors.extend(compiled.errors); - output.sources.extend(compiled.sources); - output.contracts.extend(compiled.contracts); - } else { - self.compiler_output = Some(compiled); - } + if let Some(output) = compiler_output { + self.extend_output(output); } } + pub fn extend_output(&mut self, compiled: CompilerOutput) { + if let Some(output) = self.compiler_output.as_mut() { + output.errors.extend(compiled.errors); + output.sources.extend(compiled.sources); + output.contracts.extend(compiled.contracts); + } else { + self.compiler_output = Some(compiled); + } + } + + pub fn extend_artifacts(&mut self, artifacts: BTreeMap) { + self.artifacts.extend(artifacts); + } + /// Whether this type does not contain compiled contracts pub fn is_unchanged(&self) -> bool { !self.has_compiled_contracts()