From 4d17fb3af6eeb37f4cd2f8c79873700e7d0d6587 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 6 Apr 2023 06:36:21 +0000 Subject: [PATCH] CI fix: move download web3signer binary out of build script (#4163) ## Issue Addressed Attempt to fix #3812 ## Proposed Changes Move web3signer binary download script out of build script to avoid downloading unless necessary. If this works, it should also reduce the build time for all jobs that runs compilation. --- Cargo.lock | 2 + testing/web3signer_tests/Cargo.toml | 8 +--- .../{build.rs => src/get_web3signer.rs} | 11 ------ testing/web3signer_tests/src/lib.rs | 39 ++++++++++++++++--- 4 files changed, 38 insertions(+), 22 deletions(-) rename testing/web3signer_tests/{build.rs => src/get_web3signer.rs} (88%) diff --git a/Cargo.lock b/Cargo.lock index 7a67b77bf9e..08b7e2a8081 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9227,6 +9227,8 @@ dependencies = [ "eth2_network_config", "exit-future", "futures", + "lazy_static", + "parking_lot 0.12.1", "reqwest", "serde", "serde_derive", diff --git a/testing/web3signer_tests/Cargo.toml b/testing/web3signer_tests/Cargo.toml index 8ce58300629..c0fbf667236 100644 --- a/testing/web3signer_tests/Cargo.toml +++ b/testing/web3signer_tests/Cargo.toml @@ -3,8 +3,6 @@ name = "web3signer_tests" version = "0.1.0" edition = "2021" -build = "build.rs" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -27,9 +25,7 @@ serde = "1.0.116" serde_derive = "1.0.116" serde_yaml = "0.8.13" eth2_network_config = { path = "../../common/eth2_network_config" } - -[build-dependencies] -tokio = { version = "1.14.0", features = ["rt-multi-thread", "macros"] } -reqwest = { version = "0.11.0", features = ["json","stream"] } serde_json = "1.0.58" zip = "0.5.13" +lazy_static = "1.4.0" +parking_lot = "0.12.0" \ No newline at end of file diff --git a/testing/web3signer_tests/build.rs b/testing/web3signer_tests/src/get_web3signer.rs similarity index 88% rename from testing/web3signer_tests/build.rs rename to testing/web3signer_tests/src/get_web3signer.rs index a55c39376a4..800feb204ae 100644 --- a/testing/web3signer_tests/build.rs +++ b/testing/web3signer_tests/src/get_web3signer.rs @@ -15,17 +15,6 @@ use zip::ZipArchive; /// Use `Some("21.8.1")` to download a specific version. const FIXED_VERSION_STRING: Option<&str> = None; -#[tokio::main] -async fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - - // Read a Github API token from the environment. This is intended to prevent rate-limits on CI. - // We use a name that is unlikely to accidentally collide with anything the user has configured. - let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN"); - - download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await; -} - pub async fn download_binary(dest_dir: PathBuf, github_token: &str) { let version_file = dest_dir.join("version"); diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index 16bffd04f96..dd17ae23b15 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -9,16 +9,21 @@ //! - Lighthouse can issue valid requests to Web3Signer. //! - The signatures generated by Web3Signer are identical to those which Lighthouse generates. //! -//! There is a build script in this crate which obtains the latest version of Web3Signer and makes -//! it available via the `OUT_DIR`. +//! There is a `download_binary` function in the `get_web3signer` module which obtains the latest version of Web3Signer and makes +//! it available via the `TEMP_DIR`. +#![cfg(all(test, unix, not(debug_assertions)))] + +mod get_web3signer; -#[cfg(all(test, unix, not(debug_assertions)))] mod tests { + use crate::get_web3signer::download_binary; use account_utils::validator_definitions::{ SigningDefinition, ValidatorDefinition, ValidatorDefinitions, Web3SignerDefinition, }; use eth2_keystore::KeystoreBuilder; use eth2_network_config::Eth2NetworkConfig; + use lazy_static::lazy_static; + use parking_lot::Mutex; use reqwest::Client; use serde::Serialize; use slot_clock::{SlotClock, TestingSlotClock}; @@ -31,7 +36,8 @@ mod tests { use std::sync::Arc; use std::time::{Duration, Instant}; use task_executor::TaskExecutor; - use tempfile::TempDir; + use tempfile::{tempdir, TempDir}; + use tokio::sync::OnceCell; use tokio::time::sleep; use types::*; use url::Url; @@ -51,6 +57,13 @@ mod tests { /// debugging. const SUPPRESS_WEB3SIGNER_LOGS: bool = true; + lazy_static! { + static ref TEMP_DIR: Arc> = Arc::new(Mutex::new( + tempdir().expect("Failed to create temporary directory") + )); + static ref GET_WEB3SIGNER_BIN: OnceCell<()> = OnceCell::new(); + } + type E = MainnetEthSpec; /// This marker trait is implemented for objects that we wish to compare to ensure Web3Signer @@ -99,7 +112,10 @@ mod tests { /// The location of the Web3Signer binary generated by the build script. fn web3signer_binary() -> PathBuf { - PathBuf::from(env::var("OUT_DIR").unwrap()) + TEMP_DIR + .lock() + .path() + .to_path_buf() .join("web3signer") .join("bin") .join("web3signer") @@ -143,6 +159,19 @@ mod tests { impl Web3SignerRig { pub async fn new(network: &str, listen_address: &str, listen_port: u16) -> Self { + GET_WEB3SIGNER_BIN + .get_or_init(|| async { + // Read a Github API token from the environment. This is intended to prevent rate-limits on CI. + // We use a name that is unlikely to accidentally collide with anything the user has configured. + let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN"); + download_binary( + TEMP_DIR.lock().path().to_path_buf(), + github_token.as_deref().unwrap_or(""), + ) + .await; + }) + .await; + let keystore_dir = TempDir::new().unwrap(); let keypair = testing_keypair(); let keystore =