diff --git a/Cargo.lock b/Cargo.lock
index 7a67b77bf9e..08b7e2a8081 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9227,6 +9227,8 @@ dependencies = [
  "eth2_network_config",
  "exit-future",
  "futures",
+ "lazy_static",
+ "parking_lot 0.12.1",
  "reqwest",
  "serde",
  "serde_derive",
diff --git a/testing/web3signer_tests/Cargo.toml b/testing/web3signer_tests/Cargo.toml
index 8ce58300629..c0fbf667236 100644
--- a/testing/web3signer_tests/Cargo.toml
+++ b/testing/web3signer_tests/Cargo.toml
@@ -3,8 +3,6 @@ name = "web3signer_tests"
 version = "0.1.0"
 edition = "2021"
 
-build = "build.rs"
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -27,9 +25,7 @@ serde = "1.0.116"
 serde_derive = "1.0.116"
 serde_yaml = "0.8.13"
 eth2_network_config = { path = "../../common/eth2_network_config" }
-
-[build-dependencies]
-tokio = { version = "1.14.0", features = ["rt-multi-thread", "macros"] }
-reqwest = { version = "0.11.0", features = ["json","stream"] }
 serde_json = "1.0.58"
 zip = "0.5.13"
+lazy_static = "1.4.0"
+parking_lot = "0.12.0"
\ No newline at end of file
diff --git a/testing/web3signer_tests/build.rs b/testing/web3signer_tests/src/get_web3signer.rs
similarity index 88%
rename from testing/web3signer_tests/build.rs
rename to testing/web3signer_tests/src/get_web3signer.rs
index a55c39376a4..800feb204ae 100644
--- a/testing/web3signer_tests/build.rs
+++ b/testing/web3signer_tests/src/get_web3signer.rs
@@ -15,17 +15,6 @@ use zip::ZipArchive;
 /// Use `Some("21.8.1")` to download a specific version.
 const FIXED_VERSION_STRING: Option<&str> = None;
 
-#[tokio::main]
-async fn main() {
-    let out_dir = env::var("OUT_DIR").unwrap();
-
-    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
-    // We use a name that is unlikely to accidentally collide with anything the user has configured.
-    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
-
-    download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await;
-}
-
 pub async fn download_binary(dest_dir: PathBuf, github_token: &str) {
     let version_file = dest_dir.join("version");
 
diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs
index 16bffd04f96..dd17ae23b15 100644
--- a/testing/web3signer_tests/src/lib.rs
+++ b/testing/web3signer_tests/src/lib.rs
@@ -9,16 +9,21 @@
 //! - Lighthouse can issue valid requests to Web3Signer.
 //! - The signatures generated by Web3Signer are identical to those which Lighthouse generates.
 //!
-//! There is a build script in this crate which obtains the latest version of Web3Signer and makes
-//! it available via the `OUT_DIR`.
+//! There is a `download_binary` function in the `get_web3signer` module which obtains the latest version of Web3Signer and makes
+//! it available via the `TEMP_DIR`.
+#![cfg(all(test, unix, not(debug_assertions)))]
+
+mod get_web3signer;
 
-#[cfg(all(test, unix, not(debug_assertions)))]
 mod tests {
+    use crate::get_web3signer::download_binary;
     use account_utils::validator_definitions::{
         SigningDefinition, ValidatorDefinition, ValidatorDefinitions, Web3SignerDefinition,
     };
     use eth2_keystore::KeystoreBuilder;
     use eth2_network_config::Eth2NetworkConfig;
+    use lazy_static::lazy_static;
+    use parking_lot::Mutex;
     use reqwest::Client;
     use serde::Serialize;
     use slot_clock::{SlotClock, TestingSlotClock};
@@ -31,7 +36,8 @@ mod tests {
     use std::sync::Arc;
     use std::time::{Duration, Instant};
     use task_executor::TaskExecutor;
-    use tempfile::TempDir;
+    use tempfile::{tempdir, TempDir};
+    use tokio::sync::OnceCell;
     use tokio::time::sleep;
     use types::*;
     use url::Url;
@@ -51,6 +57,13 @@ mod tests {
     /// debugging.
     const SUPPRESS_WEB3SIGNER_LOGS: bool = true;
 
+    lazy_static! {
+        static ref TEMP_DIR: Arc<Mutex<TempDir>> = Arc::new(Mutex::new(
+            tempdir().expect("Failed to create temporary directory")
+        ));
+        static ref GET_WEB3SIGNER_BIN: OnceCell<()> = OnceCell::new();
+    }
+
     type E = MainnetEthSpec;
 
     /// This marker trait is implemented for objects that we wish to compare to ensure Web3Signer
@@ -99,7 +112,10 @@ mod tests {
 
     /// The location of the Web3Signer binary generated by the build script.
     fn web3signer_binary() -> PathBuf {
-        PathBuf::from(env::var("OUT_DIR").unwrap())
+        TEMP_DIR
+            .lock()
+            .path()
+            .to_path_buf()
             .join("web3signer")
             .join("bin")
             .join("web3signer")
@@ -143,6 +159,19 @@ mod tests {
 
     impl Web3SignerRig {
         pub async fn new(network: &str, listen_address: &str, listen_port: u16) -> Self {
+            GET_WEB3SIGNER_BIN
+                .get_or_init(|| async {
+                    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
+                    // We use a name that is unlikely to accidentally collide with anything the user has configured.
+                    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
+                    download_binary(
+                        TEMP_DIR.lock().path().to_path_buf(),
+                        github_token.as_deref().unwrap_or(""),
+                    )
+                    .await;
+                })
+                .await;
+
             let keystore_dir = TempDir::new().unwrap();
             let keypair = testing_keypair();
             let keystore =