From 2e1811eeb597668aaffefaf62e1fd00f91dd0e7c Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 22 May 2023 17:52:19 -0400 Subject: [PATCH 01/14] HTTP Cache --- Cargo.lock | 93 +++++++++-------- crates/turborepo-api-client/src/lib.rs | 90 +++++++++++++++- crates/turborepo-cache/src/http.rs | 137 +++++++++++++++++++++++++ crates/turborepo-cache/src/lib.rs | 11 +- 4 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 crates/turborepo-cache/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index 4173812b14689..daa3c7415aea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -729,9 +729,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" dependencies = [ "arrayref", "arrayvec 0.7.2", @@ -1389,9 +1389,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" [[package]] name = "convert_case" @@ -1959,9 +1959,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "dav1d" @@ -2049,9 +2049,9 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" [[package]] name = "digest" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", @@ -3401,9 +3401,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.11" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -3858,9 +3858,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" [[package]] name = "lock_api" @@ -3874,10 +3874,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.18" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ + "cfg-if 1.0.0", "value-bag", ] @@ -4094,9 +4095,9 @@ dependencies = [ [[package]] name = "mimalloc-rust" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb726c8298efb4010b2c46d8050e4be36cf807b9d9e98cb112f830914fc9bbe" +checksum = "6973866e0bc6504c03a16b6817b7e70839cc8a1dbd5d6dab00c65d8034868d8b" dependencies = [ "cty", "mimalloc-rust-sys", @@ -4104,9 +4105,9 @@ dependencies = [ [[package]] name = "mimalloc-rust-sys" -version = "1.7.9-source" +version = "1.7.6-source" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6413e13241a9809f291568133eca6694572cf528c1a6175502d090adce5dd5db" +checksum = "7a50daf45336b979a202a19f53b4b382f2c4bd50f392a8dbdb4c6c56ba5dfa64" dependencies = [ "cc", "cty", @@ -4954,18 +4955,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.0" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", @@ -5383,9 +5384,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -5604,13 +5605,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +checksum = "ac6cf59af1067a3fb53fbe5c88c053764e930f932be1d71d3ffe032cbe147f59" dependencies = [ "aho-corasick 1.0.1", "memchr", - "regex-syntax 0.7.2", + "regex-syntax 0.7.0", ] [[package]] @@ -5630,9 +5631,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5" [[package]] name = "region" @@ -5903,9 +5904,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ "bitflags 1.3.2", "errno", @@ -8291,9 +8292,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", @@ -8306,7 +8307,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.45.0", ] [[package]] @@ -8321,9 +8322,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", @@ -8549,9 +8550,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -8571,9 +8572,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -9868,9 +9869,9 @@ checksum = "d70b6494226b36008c8366c288d77190b3fad2eb4c10533139c1c1f461127f1a" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-linebreak" @@ -9991,9 +9992,13 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.0" +version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +dependencies = [ + "ctor 0.1.26", + "version_check", +] [[package]] name = "vcpkg" diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index 71dfc8ffd672d..55715940c5473 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -4,8 +4,9 @@ use std::env; -use reqwest::RequestBuilder; +use reqwest::{Method, RequestBuilder, Response}; use serde::{Deserialize, Serialize}; +use url::Url; pub use crate::error::{Error, Result}; @@ -39,6 +40,13 @@ pub struct CachingStatusResponse { pub status: CachingStatus, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArtifactResponse { + pub duration: u64, + pub expected_tag: Option, + pub body: Vec, +} + /// Membership is the relationship between the logged-in user and a particular /// team #[derive(Debug, Clone, Serialize, Deserialize)] @@ -111,6 +119,11 @@ pub struct UserResponse { pub user: User, } +pub struct PreflightResponse { + location: Url, + allow_auth: bool, +} + pub struct APIClient { client: reqwest::Client, base_url: String, @@ -240,6 +253,81 @@ impl APIClient { }) } + pub async fn fetch_artifact( + &self, + hash: &str, + token: &str, + team_id: &str, + team_slug: Option<&str>, + use_preflight: bool, + ) -> Result { + let mut request_url = self.make_url(&format!("/v8/artifacts/{}", hash)); + let mut allow_auth = true; + + if use_preflight { + let preflight_response = self + .do_preflight(token, &request_url, "GET", "Authorization, User-Agent") + .await?; + + allow_auth = preflight_response.allow_auth; + request_url = preflight_response.location.to_string(); + }; + + let mut request_builder = self + .client + .get(&request_url) + .header("User-Agent", self.user_agent.clone()); + + if allow_auth { + request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); + } + + request_builder = Self::add_team_params(request_builder, team_id, team_slug); + + let response = retry::make_retryable_request(request_builder) + .await? + .error_for_status()?; + + Ok(response) + } + + pub async fn do_preflight( + &self, + token: &str, + request_url: &str, + request_method: &str, + request_headers: &str, + ) -> Result { + let request_builder = self + .client + .request(Method::OPTIONS, request_url) + .header("User-Agent", self.user_agent.clone()) + .header("Access-Control-Request-Method", request_method) + .header("Access-Control-Request-Headers", request_headers) + .header("Authorization", format!("Bearer {}", token)); + + let response = retry::make_retryable_request(request_builder).await?; + + let headers = response.headers(); + let location = if let Some(location) = headers.get("Location") { + let location = location.to_str()?; + Url::parse(location)? + } else { + response.url().clone() + }; + + let allowed_headers = headers + .get("Access-Control-Allow-Headers") + .map_or("", |h| h.to_str().unwrap_or("")); + + let allow_auth = allowed_headers.to_lowercase().contains("authorization"); + + Ok(PreflightResponse { + location, + allow_auth, + }) + } + pub fn new(base_url: impl AsRef, timeout: u64, version: &str) -> Result { let client = if timeout != 0 { reqwest::Client::builder() diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs new file mode 100644 index 0000000000000..c1688f526c6c1 --- /dev/null +++ b/crates/turborepo-cache/src/http.rs @@ -0,0 +1,137 @@ +use std::{backtrace::Backtrace, io::Write, sync::Mutex}; + +use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; +use turborepo_api_client::APIClient; + +use crate::{ + cache_archive::{CacheReader, CacheWriter}, + signature_authentication::ArtifactSignatureAuthenticator, + CacheError, +}; + +pub struct HttpCache { + client: APIClient, + signer_verifier: Option, + repo_root: AbsoluteSystemPathBuf, +} + +impl HttpCache { + pub fn new( + client: APIClient, + signer_verifier: Option, + repo_root: AbsoluteSystemPathBuf, + ) -> HttpCache { + HttpCache { + client, + signer_verifier, + repo_root, + } + } + + pub async fn put( + &self, + anchor: &AbsoluteSystemPath, + hash: String, + duration: u64, + files: Vec, + ci_constant: Option<&str>, + ) -> Result<(), CacheError> { + let mut artifact_body = Vec::new(); + self.write(&mut artifact_body, anchor, files).await?; + + let tag = self + .signer_verifier + .map(|signer| signer.generate_tag(&hash, &artifact_body)) + .transpose()?; + + self.client + .put_artifact(&hash, &artifact_body, duration, tag, ci_constant) + .await?; + + Ok(()) + } + + async fn write( + &self, + writer: impl Write, + anchor: &AbsoluteSystemPath, + files: Vec, + ) -> Result<(), CacheError> { + let mut cache_archive = CacheWriter::from_writer(writer)?; + for file in files { + cache_archive.add_file(anchor, file.as_anchored_path())?; + } + + Ok(()) + } + + pub async fn retrieve( + &self, + hash: &str, + token: &str, + team_id: &str, + team_slug: Option<&str>, + use_preflight: bool, + ) -> Result<(Vec, u64), CacheError> { + let response = self + .client + .fetch_artifact(hash, token, team_id, team_slug, use_preflight) + .await?; + + let duration = if let Some(duration) = response.headers().get("x-artifact-duration") { + let duration = duration + .to_str() + .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))?; + duration + .parse::() + .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))? + } else { + 0 + }; + + let body = if let Some(signer_verifier) = &self.signer_verifier { + let expected_tag = response + .headers() + .get("x-artifact-tag") + .ok_or(CacheError::ArtifactTagMissing(Backtrace::capture()))?; + + let expected_tag = expected_tag + .to_str() + .map_err(|_| CacheError::InvalidTag(Backtrace::capture()))? + .to_string(); + + let body = response.bytes().await.map_err(|e| { + CacheError::ApiClientError( + turborepo_api_client::Error::ReqwestError(e), + Backtrace::capture(), + ) + })?; + let is_valid = signer_verifier.validate(hash, &body, &expected_tag)?; + + if !is_valid { + return Err(CacheError::InvalidTag(Backtrace::capture())); + } + + body + } else { + response.bytes().await.map_err(|e| { + CacheError::ApiClientError( + turborepo_api_client::Error::ReqwestError(e), + Backtrace::capture(), + ) + })? + }; + + let files = Self::restore_tar(self.repo_root.as_absolute_path(), &body)?; + + Ok((files, duration)) + } + + pub(crate) fn restore_tar( + root: &AbsoluteSystemPath, + body: &[u8], + ) -> Result, CacheError> { + let mut cache_reader = CacheReader::from_reader(body, true)?; + cache_reader.restore(root) + } +} diff --git a/crates/turborepo-cache/src/lib.rs b/crates/turborepo-cache/src/lib.rs index 715a29f6fff4d..928d5fb6d363b 100644 --- a/crates/turborepo-cache/src/lib.rs +++ b/crates/turborepo-cache/src/lib.rs @@ -2,9 +2,14 @@ #![feature(provide_any)] pub mod cache_archive; +pub mod http; pub mod signature_authentication; -use std::{backtrace, backtrace::Backtrace}; +use std::{ + backtrace, + backtrace::Backtrace, + sync::{MutexGuard, PoisonError}, +}; use thiserror::Error; @@ -23,6 +28,8 @@ pub enum CacheError { InvalidTag(#[backtrace] Backtrace), #[error("cannot untar file to {0}")] InvalidFilePath(String, #[backtrace] Backtrace), + #[error("artifact verification failed: {0}")] + ApiClientError(#[from] turborepo_api_client::Error, #[backtrace] Backtrace), #[error("signing artifact failed: {0}")] SignatureError(#[from] SignatureError, #[backtrace] Backtrace), #[error("invalid duration")] @@ -41,8 +48,6 @@ pub enum CacheError { // way to display it nicely. #[error("attempted to create unsupported file type")] CreateUnsupportedFileType(#[backtrace] Backtrace), - #[error("file name is malformed: {0}")] - MalformedName(String, #[backtrace] Backtrace), #[error("tar file is malformed")] MalformedTar(#[backtrace] Backtrace), #[error("file name is not Windows-safe: {0}")] From 57798fc38f42d0788ebdccbb9fa6f7bd3ff825f7 Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 29 Jun 2023 14:57:21 -0400 Subject: [PATCH 02/14] Integrating with cache item --- .../src/cache_archive/create.rs | 21 +++++++++++++++---- .../turborepo-cache/src/cache_archive/mod.rs | 3 +++ .../src/cache_archive/restore.rs | 9 ++++---- crates/turborepo-cache/src/http.rs | 21 +++++++++---------- crates/turborepo-cache/src/lib.rs | 6 +----- 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/crates/turborepo-cache/src/cache_archive/create.rs b/crates/turborepo-cache/src/cache_archive/create.rs index acc8a03a61c30..5363c674a579a 100644 --- a/crates/turborepo-cache/src/cache_archive/create.rs +++ b/crates/turborepo-cache/src/cache_archive/create.rs @@ -11,11 +11,11 @@ use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, RelativeUnixPathBuf}; use crate::CacheError; -struct CacheWriter { - builder: tar::Builder>, +pub struct CacheWriter<'a> { + builder: tar::Builder>, } -impl CacheWriter { +impl<'a> CacheWriter<'a> { // Appends data to tar builder. fn append_data( &mut self, @@ -30,6 +30,19 @@ impl CacheWriter { Ok(self.builder.finish()?) } + pub fn from_writer(writer: impl Write + 'a, use_compression: bool) -> Result { + if use_compression { + let zw = zstd::Encoder::new(writer, 0)?.auto_finish(); + Ok(CacheWriter { + builder: tar::Builder::new(Box::new(zw)), + }) + } else { + Ok(CacheWriter { + builder: tar::Builder::new(Box::new(writer)), + }) + } + } + // Makes a new CacheArchive at the specified path // Wires up the chain of writers: // tar::Builder -> zstd::Encoder (optional) -> BufWriter -> File @@ -58,7 +71,7 @@ impl CacheWriter { } // Adds a user-cached item to the tar - fn add_file( + pub(crate) fn add_file( &mut self, anchor: &AbsoluteSystemPath, file_path: &AnchoredSystemPath, diff --git a/crates/turborepo-cache/src/cache_archive/mod.rs b/crates/turborepo-cache/src/cache_archive/mod.rs index 9ba36d6646e2e..72bc6a595ad11 100644 --- a/crates/turborepo-cache/src/cache_archive/mod.rs +++ b/crates/turborepo-cache/src/cache_archive/mod.rs @@ -4,3 +4,6 @@ mod restore; mod restore_directory; mod restore_regular; mod restore_symlink; + +pub use create::CacheWriter; +pub use restore::CacheReader; diff --git a/crates/turborepo-cache/src/cache_archive/restore.rs b/crates/turborepo-cache/src/cache_archive/restore.rs index 335046b0af01c..b9dfcd85271e7 100644 --- a/crates/turborepo-cache/src/cache_archive/restore.rs +++ b/crates/turborepo-cache/src/cache_archive/restore.rs @@ -16,13 +16,12 @@ use crate::{ CacheError, }; -pub struct CacheReader { - reader: Box, +pub struct CacheReader<'a> { + reader: Box, } -impl CacheReader { - #[cfg(test)] - pub fn new(reader: impl Read + 'static, is_compressed: bool) -> Result { +impl<'a> CacheReader<'a> { + pub fn from_reader(reader: impl Read + 'a, is_compressed: bool) -> Result { let reader: Box = if is_compressed { Box::new(zstd::Decoder::new(reader)?) } else { diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index c1688f526c6c1..37ddfff638aba 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -1,4 +1,4 @@ -use std::{backtrace::Backtrace, io::Write, sync::Mutex}; +use std::{backtrace::Backtrace, io::Write}; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; use turborepo_api_client::APIClient; @@ -32,21 +32,20 @@ impl HttpCache { &self, anchor: &AbsoluteSystemPath, hash: String, - duration: u64, files: Vec, - ci_constant: Option<&str>, ) -> Result<(), CacheError> { let mut artifact_body = Vec::new(); self.write(&mut artifact_body, anchor, files).await?; let tag = self .signer_verifier - .map(|signer| signer.generate_tag(&hash, &artifact_body)) + .as_ref() + .map(|signer| signer.generate_tag(hash.as_bytes(), &artifact_body)) .transpose()?; - self.client - .put_artifact(&hash, &artifact_body, duration, tag, ci_constant) - .await?; + // self.client + // .put_artifact(&hash, &artifact_body, duration, tag, ci_constant) + // .await?; Ok(()) } @@ -57,9 +56,9 @@ impl HttpCache { anchor: &AbsoluteSystemPath, files: Vec, ) -> Result<(), CacheError> { - let mut cache_archive = CacheWriter::from_writer(writer)?; + let mut cache_archive = CacheWriter::from_writer(writer, true)?; for file in files { - cache_archive.add_file(anchor, file.as_anchored_path())?; + cache_archive.add_file(anchor, &file)?; } Ok(()) @@ -106,7 +105,7 @@ impl HttpCache { Backtrace::capture(), ) })?; - let is_valid = signer_verifier.validate(hash, &body, &expected_tag)?; + let is_valid = signer_verifier.validate(hash.as_bytes(), &body, &expected_tag)?; if !is_valid { return Err(CacheError::InvalidTag(Backtrace::capture())); @@ -122,7 +121,7 @@ impl HttpCache { })? }; - let files = Self::restore_tar(self.repo_root.as_absolute_path(), &body)?; + let files = Self::restore_tar(&self.repo_root, &body)?; Ok((files, duration)) } diff --git a/crates/turborepo-cache/src/lib.rs b/crates/turborepo-cache/src/lib.rs index 928d5fb6d363b..60d67c717858d 100644 --- a/crates/turborepo-cache/src/lib.rs +++ b/crates/turborepo-cache/src/lib.rs @@ -5,11 +5,7 @@ pub mod cache_archive; pub mod http; pub mod signature_authentication; -use std::{ - backtrace, - backtrace::Backtrace, - sync::{MutexGuard, PoisonError}, -}; +use std::{backtrace, backtrace::Backtrace}; use thiserror::Error; From fd1f0194ce66b876d1dacfc6f0b582d1c80b9fa2 Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 29 Jun 2023 15:51:17 -0400 Subject: [PATCH 03/14] Integrated with existing code --- cli/internal/client/cache.go | 4 +- crates/turborepo-api-client/src/lib.rs | 59 +++++++++++++++++++++++- crates/turborepo-cache/src/http.rs | 8 ++-- crates/turborepo-lib/src/commands/mod.rs | 8 +++- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/cli/internal/client/cache.go b/cli/internal/client/cache.go index 11ad87a948a2a..533d89c5b8983 100644 --- a/cli/internal/client/cache.go +++ b/cli/internal/client/cache.go @@ -30,7 +30,9 @@ func (c *APIClient) PutArtifact(hash string, artifactBody []byte, duration int, requestURL := c.makeURL("/v8/artifacts/" + hash + encoded) allowAuth := true if c.usePreflight { - resp, latestRequestURL, err := c.doPreflight(requestURL, http.MethodPut, "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag") + resp, latestRequestURL, err := c.doPreflight(requestURL, + http.MethodPut, + "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag") if err != nil { return fmt.Errorf("pre-flight request failed before trying to store in HTTP cache: %w", err) } diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index 55715940c5473..91f38bb18f001 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -128,6 +128,7 @@ pub struct APIClient { client: reqwest::Client, base_url: String, user_agent: String, + use_preflight: bool, } impl APIClient { @@ -253,6 +254,56 @@ impl APIClient { }) } + pub async fn put_artifact( + &self, + hash: &str, + artifact_body: &[u8], + duration: u32, + tag: Option<&str>, + token: &str, + ) -> Result<()> { + let mut request_url = self.make_url(&format!("/v8/artifacts/{}", hash)); + let mut allow_auth = true; + + if self.use_preflight { + let preflight_response = self + .do_preflight( + token, + &request_url, + "PUT", + "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag", + ) + .await?; + + allow_auth = preflight_response.allow_auth; + request_url = preflight_response.location.to_string(); + } + + let mut request_builder = self + .client + .put(&request_url) + .header("Content-Type", "application/octet-stream") + .header("x-artifact-duration", duration.to_string()) + .header("User-Agent", self.user_agent.clone()) + .body(artifact_body.to_vec()); + + // TODO: Add CI header + + if allow_auth { + request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); + } + + if let Some(tag) = tag { + request_builder = request_builder.header("x-artifact-tag", tag); + } + + retry::make_retryable_request(request_builder) + .await? + .error_for_status()?; + + Ok(()) + } + pub async fn fetch_artifact( &self, hash: &str, @@ -328,7 +379,12 @@ impl APIClient { }) } - pub fn new(base_url: impl AsRef, timeout: u64, version: &str) -> Result { + pub fn new( + base_url: impl AsRef, + timeout: u64, + version: &str, + use_preflight: bool, + ) -> Result { let client = if timeout != 0 { reqwest::Client::builder() .timeout(std::time::Duration::from_secs(timeout)) @@ -348,6 +404,7 @@ impl APIClient { client, base_url: base_url.as_ref().to_string(), user_agent, + use_preflight, }) } diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index 37ddfff638aba..2ff1acc2af144 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -33,6 +33,8 @@ impl HttpCache { anchor: &AbsoluteSystemPath, hash: String, files: Vec, + duration: u32, + token: &str, ) -> Result<(), CacheError> { let mut artifact_body = Vec::new(); self.write(&mut artifact_body, anchor, files).await?; @@ -43,9 +45,9 @@ impl HttpCache { .map(|signer| signer.generate_tag(hash.as_bytes(), &artifact_body)) .transpose()?; - // self.client - // .put_artifact(&hash, &artifact_body, duration, tag, ci_constant) - // .await?; + self.client + .put_artifact(&hash, &artifact_body, duration, tag.as_deref(), token) + .await?; Ok(()) } diff --git a/crates/turborepo-lib/src/commands/mod.rs b/crates/turborepo-lib/src/commands/mod.rs index 4dfc02c0b471a..3bb648f068637 100644 --- a/crates/turborepo-lib/src/commands/mod.rs +++ b/crates/turborepo-lib/src/commands/mod.rs @@ -145,10 +145,16 @@ impl CommandBase { pub fn api_client(&mut self) -> Result { let repo_config = self.repo_config()?; let client_config = self.client_config()?; + let args = self.args(); let api_url = repo_config.api_url(); let timeout = client_config.remote_cache_timeout(); - Ok(APIClient::new(api_url, timeout, self.version)?) + Ok(APIClient::new( + api_url, + timeout, + self.version, + args.preflight, + )?) } pub fn daemon_file_root(&self) -> AbsoluteSystemPathBuf { From 0df0ce3a03d7eaaeced0c88df883c9f6751096dd Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 29 Jun 2023 16:36:05 -0400 Subject: [PATCH 04/14] Adding mock for HTTP cache put artifact --- Cargo.lock | 15 +++++++ crates/turborepo-cache/src/http.rs | 49 ++++++++++++++++++++- crates/turborepo-vercel-api-mock/Cargo.toml | 3 ++ crates/turborepo-vercel-api-mock/src/lib.rs | 33 +++++++++++++- 4 files changed, 96 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index daa3c7415aea3..0614ba5a70134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -549,6 +549,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-macros" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb524613be645939e280b7279f7b017f98cf7f5ef084ec374df373530e73277" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "axum-server" version = "0.4.7" @@ -10024,8 +10036,11 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "axum-macros", "axum-server", + "futures-util", "port_scanner", + "tempfile", "tokio", "turborepo-api-client", ] diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index 2ff1acc2af144..fe1552a134bc3 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -31,7 +31,7 @@ impl HttpCache { pub async fn put( &self, anchor: &AbsoluteSystemPath, - hash: String, + hash: &str, files: Vec, duration: u32, token: &str, @@ -46,7 +46,7 @@ impl HttpCache { .transpose()?; self.client - .put_artifact(&hash, &artifact_body, duration, tag.as_deref(), token) + .put_artifact(hash, &artifact_body, duration, tag.as_deref(), token) .await?; Ok(()) @@ -136,3 +136,48 @@ impl HttpCache { cache_reader.restore(root) } } + +#[cfg(test)] +mod test { + use anyhow::Result; + use tempfile::tempdir; + use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; + use turborepo_api_client::APIClient; + + use crate::http::HttpCache; + + const DEFAULT_API_URL: &str = "https://vercel.com/api"; + + struct TestFile { + path: AnchoredSystemPathBuf, + contents: &'static str, + } + + #[test_case(vec![ + TestFile { + path: AnchoredSystemPathBuf::new("package.json"), + contents: "{}" + } + ])] + fn test_round_trip(files: Vec) -> Result<()> { + let repo_root = tempdir()?; + let repo_root_path = AbsoluteSystemPath::try_from(repo_root.path())?; + + for file in &files { + let file_path = repo_root_path.resolve(&file.path); + std::fs::create_dir_all(file_path.parent().unwrap())?; + std::fs::write(file_path, file.contents)?; + } + let api_client = APIClient::new(DEFAULT_API_URL, 200, "2.0.0", true)?; + + let cache = HttpCache::new(api_client, None, repo_root_path.to_owned()); + + cache.put( + &repo_root_path, + "this-is-my-hash", + files.iter().map(|f| f.path.clone()).collect(), + 0, + "", + )?; + } +} diff --git a/crates/turborepo-vercel-api-mock/Cargo.toml b/crates/turborepo-vercel-api-mock/Cargo.toml index f9394af6132fa..bd36d9cc69ef3 100644 --- a/crates/turborepo-vercel-api-mock/Cargo.toml +++ b/crates/turborepo-vercel-api-mock/Cargo.toml @@ -9,7 +9,10 @@ license = "MPL-2.0" [dependencies] anyhow = { workspace = true } axum = { workspace = true } +axum-macros = "0.3.7" axum-server = "0.4.7" +futures-util = "0.3.28" port_scanner = { workspace = true } +tempfile.workspace = true tokio = { workspace = true, features = ["full"] } turborepo-api-client = { workspace = true } diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index af8d2749b38fb..d816abb57ef73 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -1,7 +1,13 @@ -use std::net::SocketAddr; +use std::{fs::OpenOptions, io::Write, net::SocketAddr, sync::Arc}; use anyhow::Result; -use axum::{routing::get, Json, Router}; +use axum::{ + extract::{BodyStream, Path}, + http::StatusCode, + routing::{get, put}, + Json, Router, +}; +use futures_util::StreamExt; use turborepo_api_client::{ CachingStatus, CachingStatusResponse, Membership, Role, Space, SpacesResponse, Team, TeamsResponse, User, UserResponse, VerificationResponse, @@ -25,6 +31,7 @@ pub const EXPECTED_SSO_TEAM_ID: &str = "expected_sso_team_id"; pub const EXPECTED_SSO_TEAM_SLUG: &str = "expected_sso_team_slug"; pub async fn start_test_server(port: u16) -> Result<()> { + let tempdir = Arc::new(tempfile::tempdir()?); let app = Router::new() .route( "/v2/user", @@ -82,7 +89,29 @@ pub async fn start_test_server(port: u16) -> Result<()> { team_id: Some(EXPECTED_SSO_TEAM_ID.to_string()), }) }), + ) + .route( + "/v8/artifacts/:hash", + put( + |Path(hash): Path, mut body: BodyStream| async move { + let root_path = tempdir.path(); + let file_path = root_path.join(&hash); + let mut file = OpenOptions::new() + .append(true) + .create(true) + .open(&file_path) + .unwrap(); + + while let Some(item) = body.next().await { + let chunk = item.unwrap(); + file.write_all(&chunk).unwrap(); + } + + (StatusCode::CREATED, Json(hash)) + }, + ), ); + let addr = SocketAddr::from(([127, 0, 0, 1], port)); // We print the port so integration tests can use it println!("{}", port); From 3a5f9499768d965f2daf3c347464259c6e5cefe7 Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 29 Jun 2023 18:57:14 -0400 Subject: [PATCH 05/14] got round trip testing to work --- Cargo.lock | 4 ++ crates/turborepo-cache/Cargo.toml | 3 ++ .../src/cache_archive/create.rs | 1 + .../src/cache_archive/restore.rs | 4 +- crates/turborepo-cache/src/http.rs | 43 ++++++++++++------- crates/turborepo-vercel-api-mock/src/lib.rs | 11 +++++ 6 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0614ba5a70134..81f1ab1bdce09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9590,6 +9590,8 @@ dependencies = [ "os_str_bytes", "path-clean 1.0.1", "petgraph", + "port_scanner", + "ring", "serde", "serde_json", "sha2", @@ -9597,9 +9599,11 @@ dependencies = [ "tempfile", "test-case", "thiserror", + "tokio", "tracing", "turbopath", "turborepo-api-client", + "vercel-api-mock", "zstd", ] diff --git a/crates/turborepo-cache/Cargo.toml b/crates/turborepo-cache/Cargo.toml index a9ed766fa6d32..e1159baeb23bc 100644 --- a/crates/turborepo-cache/Cargo.toml +++ b/crates/turborepo-cache/Cargo.toml @@ -13,8 +13,10 @@ rustls-tls = ["turborepo-api-client/rustls-tls"] [dev-dependencies] anyhow = { workspace = true, features = ["backtrace"] } libc = "0.2.146" +port_scanner = { workspace = true } tempfile = { workspace = true } test-case = { workspace = true } +vercel-api-mock = { workspace = true } [dependencies] base64 = "0.21.0" @@ -33,6 +35,7 @@ serde_json = { workspace = true } sha2 = { workspace = true } tar = "0.4.38" thiserror = { workspace = true } +tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } turbopath = { workspace = true } turborepo-api-client = { workspace = true } diff --git a/crates/turborepo-cache/src/cache_archive/create.rs b/crates/turborepo-cache/src/cache_archive/create.rs index 5363c674a579a..00ab971182471 100644 --- a/crates/turborepo-cache/src/cache_archive/create.rs +++ b/crates/turborepo-cache/src/cache_archive/create.rs @@ -127,6 +127,7 @@ impl<'a> CacheWriter<'a> { header.set_entry_type(EntryType::Directory); } else if file_info.is_file() { header.set_entry_type(EntryType::Regular); + header.set_size(file_info.len()); } else { // Throw an error if trying to create a cache that contains a type we don't // support. diff --git a/crates/turborepo-cache/src/cache_archive/restore.rs b/crates/turborepo-cache/src/cache_archive/restore.rs index b9dfcd85271e7..70d382e3d9738 100644 --- a/crates/turborepo-cache/src/cache_archive/restore.rs +++ b/crates/turborepo-cache/src/cache_archive/restore.rs @@ -341,7 +341,7 @@ mod tests { for (tar_bytes, is_compressed) in [(&uncompressed_tar[..], false), (&compressed_tar[..], true)] { - let mut cache_reader = CacheReader::new(tar_bytes, is_compressed)?; + let mut cache_reader = CacheReader::from_reader(&tar_bytes[..], is_compressed)?; let output_dir = tempdir()?; let output_dir_path = output_dir.path().to_string_lossy(); let anchor = AbsoluteSystemPath::new(&output_dir_path)?; @@ -364,7 +364,7 @@ mod tests { for (tar_bytes, is_compressed) in [(&uncompressed_tar[..], false), (&compressed_tar[..], true)] { - let mut cache_reader = CacheReader::new(tar_bytes, is_compressed)?; + let mut cache_reader = CacheReader::from_reader(&tar_bytes[..], is_compressed)?; let output_dir = tempdir()?; let output_dir_path = output_dir.path().to_string_lossy(); let anchor = AbsoluteSystemPath::new(&output_dir_path)?; diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index fe1552a134bc3..c2249995f3a8a 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -141,13 +141,13 @@ impl HttpCache { mod test { use anyhow::Result; use tempfile::tempdir; - use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; + use test_case::test_case; + use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; use turborepo_api_client::APIClient; + use vercel_api_mock::start_test_server; use crate::http::HttpCache; - const DEFAULT_API_URL: &str = "https://vercel.com/api"; - struct TestFile { path: AnchoredSystemPathBuf, contents: &'static str, @@ -155,29 +155,42 @@ mod test { #[test_case(vec![ TestFile { - path: AnchoredSystemPathBuf::new("package.json"), - contents: "{}" + path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(), + contents: "hello world" } ])] - fn test_round_trip(files: Vec) -> Result<()> { + #[tokio::test] + async fn test_round_trip(files: Vec) -> Result<()> { + let port = port_scanner::request_open_port().unwrap(); + let handle = tokio::spawn(start_test_server(port)); + let repo_root = tempdir()?; - let repo_root_path = AbsoluteSystemPath::try_from(repo_root.path())?; + let repo_root_path = AbsoluteSystemPathBuf::try_from(repo_root.path())?; for file in &files { let file_path = repo_root_path.resolve(&file.path); std::fs::create_dir_all(file_path.parent().unwrap())?; std::fs::write(file_path, file.contents)?; } - let api_client = APIClient::new(DEFAULT_API_URL, 200, "2.0.0", true)?; + let api_client = APIClient::new(&format!("http://localhost:{}", port), 200, "2.0.0", true)?; let cache = HttpCache::new(api_client, None, repo_root_path.to_owned()); - cache.put( - &repo_root_path, - "this-is-my-hash", - files.iter().map(|f| f.path.clone()).collect(), - 0, - "", - )?; + cache + .put( + &repo_root_path, + "this-is-my-hash", + files.iter().map(|f| f.path.clone()).collect(), + 0, + "", + ) + .await?; + + cache + .retrieve("this-is-my-hash", "", "", None, false) + .await?; + + handle.abort(); + Ok(()) } } diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index d816abb57ef73..d36e2116d2c5b 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -32,6 +32,7 @@ pub const EXPECTED_SSO_TEAM_SLUG: &str = "expected_sso_team_slug"; pub async fn start_test_server(port: u16) -> Result<()> { let tempdir = Arc::new(tempfile::tempdir()?); + let tempdir2 = tempdir.clone(); let app = Router::new() .route( "/v2/user", @@ -110,6 +111,16 @@ pub async fn start_test_server(port: u16) -> Result<()> { (StatusCode::CREATED, Json(hash)) }, ), + ) + .route( + "/v8/artifacts/:hash", + get(|Path(hash): Path| async move { + let root_path = tempdir2.path(); + let file_path = root_path.join(&hash); + let buffer = std::fs::read(file_path).unwrap(); + + (StatusCode::OK, buffer) + }), ); let addr = SocketAddr::from(([127, 0, 0, 1], port)); From 9868b684122a632094415da75253e1d50adda821 Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 30 Jun 2023 11:36:15 -0400 Subject: [PATCH 06/14] More tests --- crates/turborepo-cache/src/http.rs | 48 +++++++++++++++++---- crates/turborepo-vercel-api-mock/src/lib.rs | 26 +++++++++-- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index c2249995f3a8a..843a931f82281 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -73,7 +73,7 @@ impl HttpCache { team_id: &str, team_slug: Option<&str>, use_preflight: bool, - ) -> Result<(Vec, u64), CacheError> { + ) -> Result<(Vec, u32), CacheError> { let response = self .client .fetch_artifact(hash, token, team_id, team_slug, use_preflight) @@ -84,7 +84,7 @@ impl HttpCache { .to_str() .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))?; duration - .parse::() + .parse::() .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))? } else { 0 @@ -158,9 +158,33 @@ mod test { path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(), contents: "hello world" } - ])] + ], 58, "Faces Places")] + #[test_case(vec![ + TestFile { + path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(), + contents: "Days of Heaven" + }, + TestFile { + path: AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(), + contents: "Badlands" + } + ], 1284, "Cleo from 5 to 7")] + #[test_case(vec![ + TestFile { + path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(), + contents: "Days of Heaven" + }, + TestFile { + path: AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(), + contents: "Badlands" + }, + TestFile { + path: AnchoredSystemPathBuf::from_raw("src/main.js").unwrap(), + contents: "Tree of Life" + } + ], 12845, "The Gleaners and I")] #[tokio::test] - async fn test_round_trip(files: Vec) -> Result<()> { + async fn test_round_trip(files: Vec, duration: u32, hash: &str) -> Result<()> { let port = port_scanner::request_open_port().unwrap(); let handle = tokio::spawn(start_test_server(port)); @@ -172,6 +196,7 @@ mod test { std::fs::create_dir_all(file_path.parent().unwrap())?; std::fs::write(file_path, file.contents)?; } + let api_client = APIClient::new(&format!("http://localhost:{}", port), 200, "2.0.0", true)?; let cache = HttpCache::new(api_client, None, repo_root_path.to_owned()); @@ -179,16 +204,21 @@ mod test { cache .put( &repo_root_path, - "this-is-my-hash", + hash, files.iter().map(|f| f.path.clone()).collect(), - 0, + duration, "", ) .await?; - cache - .retrieve("this-is-my-hash", "", "", None, false) - .await?; + let (received_files, received_duration) = cache.retrieve(hash, "", "", None, false).await?; + + assert_eq!(received_duration, duration); + for (test_file, received_file) in files.iter().zip(received_files) { + assert_eq!(received_file, test_file.path); + let file_path = repo_root_path.resolve(&received_file); + assert_eq!(std::fs::read_to_string(file_path)?, test_file.contents); + } handle.abort(); Ok(()) diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index d36e2116d2c5b..855d8ee4a76b8 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -1,13 +1,14 @@ -use std::{fs::OpenOptions, io::Write, net::SocketAddr, sync::Arc}; +use std::{collections::HashMap, fs::OpenOptions, io::Write, net::SocketAddr, sync::Arc}; use anyhow::Result; use axum::{ extract::{BodyStream, Path}, - http::StatusCode, + http::{HeaderMap, HeaderValue, StatusCode}, routing::{get, put}, Json, Router, }; use futures_util::StreamExt; +use tokio::sync::Mutex; use turborepo_api_client::{ CachingStatus, CachingStatusResponse, Membership, Role, Space, SpacesResponse, Team, TeamsResponse, User, UserResponse, VerificationResponse, @@ -31,6 +32,8 @@ pub const EXPECTED_SSO_TEAM_ID: &str = "expected_sso_team_id"; pub const EXPECTED_SSO_TEAM_SLUG: &str = "expected_sso_team_slug"; pub async fn start_test_server(port: u16) -> Result<()> { + let durations = Arc::new(Mutex::new(HashMap::new())); + let durations2 = durations.clone(); let tempdir = Arc::new(tempfile::tempdir()?); let tempdir2 = tempdir.clone(); let app = Router::new() @@ -94,7 +97,7 @@ pub async fn start_test_server(port: u16) -> Result<()> { .route( "/v8/artifacts/:hash", put( - |Path(hash): Path, mut body: BodyStream| async move { + |Path(hash): Path, headers: HeaderMap, mut body: BodyStream| async move { let root_path = tempdir.path(); let file_path = root_path.join(&hash); let mut file = OpenOptions::new() @@ -103,6 +106,15 @@ pub async fn start_test_server(port: u16) -> Result<()> { .open(&file_path) .unwrap(); + let duration = headers + .get("x-artifact-duration") + .and_then(|header_value| header_value.to_str().ok()) + .and_then(|duration| duration.parse::().ok()) + .unwrap_or(0); + + let mut durations_map = durations.lock().await; + durations_map.insert(hash.clone(), duration); + while let Some(item) = body.next().await { let chunk = item.unwrap(); file.write_all(&chunk).unwrap(); @@ -118,8 +130,14 @@ pub async fn start_test_server(port: u16) -> Result<()> { let root_path = tempdir2.path(); let file_path = root_path.join(&hash); let buffer = std::fs::read(file_path).unwrap(); + let duration = durations2.lock().await.remove(&hash).unwrap_or(0); + let mut headers = HeaderMap::new(); + headers.insert( + "x-artifact-duration", + HeaderValue::from_str(&duration.to_string()).unwrap(), + ); - (StatusCode::OK, buffer) + (headers, buffer) }), ); From eb7537ebd1370c04f03649eefd36a936a1ee3da0 Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 30 Jun 2023 14:42:01 -0400 Subject: [PATCH 07/14] Fixing linker errors --- cli/internal/ffi/ffi.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go index 7b6eaa6e5d250..09e1d5feaded3 100644 --- a/cli/internal/ffi/ffi.go +++ b/cli/internal/ffi/ffi.go @@ -7,12 +7,12 @@ package ffi // #include "bindings.h" // -// #cgo darwin,arm64 LDFLAGS: -L${SRCDIR} -lturborepo_ffi_darwin_arm64 -lz -liconv -framework Security -// #cgo darwin,amd64 LDFLAGS: -L${SRCDIR} -lturborepo_ffi_darwin_amd64 -lz -liconv -framework Security -// #cgo linux,arm64,staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_arm64 -lunwind -// #cgo linux,amd64,staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_amd64 -lunwind -// #cgo linux,arm64,!staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_arm64 -lz -// #cgo linux,amd64,!staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_amd64 -lz +// #cgo darwin,arm64 LDFLAGS: -L${SRCDIR} -lturborepo_ffi_darwin_arm64 -lz -liconv -framework Security -framework CoreFoundation +// #cgo darwin,amd64 LDFLAGS: -L${SRCDIR} -lturborepo_ffi_darwin_amd64 -lz -liconv -framework Security -framework CoreFoundation +// #cgo linux,arm64,staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_arm64 -lunwind -lm +// #cgo linux,amd64,staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_amd64 -lunwind -lm +// #cgo linux,arm64,!staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_arm64 -lz -lm +// #cgo linux,amd64,!staticbinary LDFLAGS: -L${SRCDIR} -lturborepo_ffi_linux_amd64 -lz -lm // #cgo windows,amd64 LDFLAGS: -L${SRCDIR} -lturborepo_ffi_windows_amd64 -lole32 -lbcrypt -lws2_32 -luserenv -lntdll import "C" From f2f6a22f4281fdc316c9262edf3966dfdaf8f038 Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 6 Jul 2023 12:18:27 -0400 Subject: [PATCH 08/14] Add exists function --- crates/turborepo-api-client/src/lib.rs | 30 +++++++++++++- crates/turborepo-cache/src/http.rs | 45 ++++++++++++++++----- crates/turborepo-vercel-api-mock/src/lib.rs | 20 ++++++++- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index 91f38bb18f001..795369afe7d99 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -4,7 +4,8 @@ use std::env; -use reqwest::{Method, RequestBuilder, Response}; +pub use reqwest::Response; +use reqwest::{Method, RequestBuilder}; use serde::{Deserialize, Serialize}; use url::Url; @@ -311,6 +312,31 @@ impl APIClient { team_id: &str, team_slug: Option<&str>, use_preflight: bool, + ) -> Result { + self.get_artifact(hash, token, team_id, team_slug, use_preflight, Method::GET) + .await + } + + pub async fn artifact_exists( + &self, + hash: &str, + token: &str, + team_id: &str, + team_slug: Option<&str>, + use_preflight: bool, + ) -> Result { + self.get_artifact(hash, token, team_id, team_slug, use_preflight, Method::HEAD) + .await + } + + async fn get_artifact( + &self, + hash: &str, + token: &str, + team_id: &str, + team_slug: Option<&str>, + use_preflight: bool, + method: Method, ) -> Result { let mut request_url = self.make_url(&format!("/v8/artifacts/{}", hash)); let mut allow_auth = true; @@ -326,7 +352,7 @@ impl APIClient { let mut request_builder = self .client - .get(&request_url) + .request(method, request_url) .header("User-Agent", self.user_agent.clone()); if allow_auth { diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index 843a931f82281..9b26069e0adad 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -1,7 +1,7 @@ use std::{backtrace::Backtrace, io::Write}; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; -use turborepo_api_client::APIClient; +use turborepo_api_client::{APIClient, Response}; use crate::{ cache_archive::{CacheReader, CacheWriter}, @@ -66,29 +66,52 @@ impl HttpCache { Ok(()) } - pub async fn retrieve( + async fn exists( &self, hash: &str, token: &str, team_id: &str, team_slug: Option<&str>, use_preflight: bool, - ) -> Result<(Vec, u32), CacheError> { + ) -> Result { let response = self .client - .fetch_artifact(hash, token, team_id, team_slug, use_preflight) + .artifact_exists(hash, token, team_id, team_slug, use_preflight) .await?; - let duration = if let Some(duration) = response.headers().get("x-artifact-duration") { + let duration = Self::get_duration_from_response(&response)?; + + Ok(duration) + } + + fn get_duration_from_response(response: &Response) -> Result { + if let Some(duration) = response.headers().get("x-artifact-duration") { let duration = duration .to_str() .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))?; + duration .parse::() - .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))? + .map_err(|_| CacheError::InvalidDuration(Backtrace::capture())) } else { - 0 - }; + Ok(0) + } + } + + pub async fn retrieve( + &self, + hash: &str, + token: &str, + team_id: &str, + team_slug: Option<&str>, + use_preflight: bool, + ) -> Result<(Vec, u32), CacheError> { + let response = self + .client + .fetch_artifact(hash, token, team_id, team_slug, use_preflight) + .await?; + + let duration = Self::get_duration_from_response(&response)?; let body = if let Some(signer_verifier) = &self.signer_verifier { let expected_tag = response @@ -211,9 +234,13 @@ mod test { ) .await?; - let (received_files, received_duration) = cache.retrieve(hash, "", "", None, false).await?; + let received_duration = cache.exists(hash, "", "", None, false).await; + assert!(received_duration.is_ok()); + assert_eq!(received_duration.unwrap(), duration); + let (received_files, received_duration) = cache.retrieve(hash, "", "", None, false).await?; assert_eq!(received_duration, duration); + for (test_file, received_file) in files.iter().zip(received_files) { assert_eq!(received_file, test_file.path); let file_path = repo_root_path.resolve(&received_file); diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index 855d8ee4a76b8..7fa6d8e47fd13 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -4,7 +4,7 @@ use anyhow::Result; use axum::{ extract::{BodyStream, Path}, http::{HeaderMap, HeaderValue, StatusCode}, - routing::{get, put}, + routing::{get, head, put}, Json, Router, }; use futures_util::StreamExt; @@ -34,6 +34,7 @@ pub const EXPECTED_SSO_TEAM_SLUG: &str = "expected_sso_team_slug"; pub async fn start_test_server(port: u16) -> Result<()> { let durations = Arc::new(Mutex::new(HashMap::new())); let durations2 = durations.clone(); + let durations3 = durations.clone(); let tempdir = Arc::new(tempfile::tempdir()?); let tempdir2 = tempdir.clone(); let app = Router::new() @@ -130,8 +131,9 @@ pub async fn start_test_server(port: u16) -> Result<()> { let root_path = tempdir2.path(); let file_path = root_path.join(&hash); let buffer = std::fs::read(file_path).unwrap(); - let duration = durations2.lock().await.remove(&hash).unwrap_or(0); + let duration = durations2.lock().await.get(&hash).cloned().unwrap_or(0); let mut headers = HeaderMap::new(); + headers.insert( "x-artifact-duration", HeaderValue::from_str(&duration.to_string()).unwrap(), @@ -139,6 +141,20 @@ pub async fn start_test_server(port: u16) -> Result<()> { (headers, buffer) }), + ) + .route( + "/v8/artifacts/:hash", + head(|Path(hash): Path| async move { + let duration = durations3.lock().await.get(&hash).cloned().unwrap_or(0); + let mut headers = HeaderMap::new(); + + headers.insert( + "x-artifact-duration", + HeaderValue::from_str(&duration.to_string()).unwrap(), + ); + + headers + }), ); let addr = SocketAddr::from(([127, 0, 0, 1], port)); From 229f480e008c02670f23c2485ae043c4aa1ac0a8 Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 7 Jul 2023 14:54:15 -0400 Subject: [PATCH 09/14] Updating tests Co-authored-by: Nathan Hammond --- Cargo.lock | 4 ++-- .../src/cache_archive/create.rs | 18 +++++++++--------- crates/turborepo-cache/src/http.rs | 2 +- crates/turborepo-vercel-api-mock/Cargo.toml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81f1ab1bdce09..ad2016d6dfd67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6725,9 +6725,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "supports-color" diff --git a/crates/turborepo-cache/src/cache_archive/create.rs b/crates/turborepo-cache/src/cache_archive/create.rs index 00ab971182471..1ea5139255c2b 100644 --- a/crates/turborepo-cache/src/cache_archive/create.rs +++ b/crates/turborepo-cache/src/cache_archive/create.rs @@ -242,9 +242,9 @@ mod tests { file_type: FileType::File, } ], - "db05810ef8714bc849a27d2b78a267c03862cd5259a5c7fb916e92a1ef912da68a4c92032d8e984e241e12fb85a4b41574009922d740c7e66faf50a00682003c", - "db05810ef8714bc849a27d2b78a267c03862cd5259a5c7fb916e92a1ef912da68a4c92032d8e984e241e12fb85a4b41574009922d740c7e66faf50a00682003c", - "224fda5e3b1db1e4a7ede1024e09ea70add3243ce1227d28b3f8fc40bca98e14d381efe4e8affc4fef8eb21b4ff42753f9923aac60038680602c117b15748ca1", + "bf0b4bf722f8d845dce7627606ab8af30bb6454d7c0379219e4c036a484960fe78e3d98e29ca0bac9b69b858d446b89d2d691c524e2884389032be799b6699f6", + "bf0b4bf722f8d845dce7627606ab8af30bb6454d7c0379219e4c036a484960fe78e3d98e29ca0bac9b69b858d446b89d2d691c524e2884389032be799b6699f6", + "4f1357753cceec5df1c8a36110ce256f3e8c5c1f62cab3283013b6266d6e97b3884711ccdd45462a4607bee7ac7a8e414d0acea4672a9f0306bcf364281edc2f", None )] #[test_case( @@ -270,9 +270,9 @@ mod tests { file_type: FileType::File, } ], - "7cb91627c62368cfa15160f9f018de3320ee0cf267625d37265d162ae3b0dea64b8126aac9769922796e3eb864189efd7c5555c4eea8999c91cbbbe695852111", - "04f27e900a4a189cf60ce21e1864ac3f77c3bc9276026a94329a5314e20a3f2671e2ac949025840f46dc9fe72f9f566f1f2c0848a3f203ba77564fae204e886c", - "1a618c123f9f09bbca9052121d13eea3192fa3addc61eb11f6dcb794f1093abba204510d126ca1f974d5db9a6e728c1e5d3b7c099faf904e494476277178d657", + "2e6febdd2e8180f91f481ae58510e4afd3f071e66b7b64d82616ebb2d2d560b9a8a814e41f723cdaa5faec90405818421d590fcf8e617df0aabaa6fc61427d4f", + "0ece16efdb0b7e2a087e622ed52f29f21a4c080d77c31c4ed940b57dcdcb1f60b910d15232c0a2747325c22dadbfd069f15de969626dc49746be2d4b9b22e239", + "2e8ad9651964faa76082306dc95bff86fa0db821681e7a8acb982244ce0a9375417e867c3a9cb82f70bc6f03c7fb085e402712d3e9f27b980d5a0c22e086f4e2", None )] #[test_case( @@ -288,9 +288,9 @@ mod tests { file_type: FileType::File, }, ], - "919de777e4d43eb072939d2e0664f9df533bd24ec357eacab83dcb8a64e2723f3ee5ecb277d1cf24538339fe06d210563188052d08dab146a8463fdb6898d655", - "919de777e4d43eb072939d2e0664f9df533bd24ec357eacab83dcb8a64e2723f3ee5ecb277d1cf24538339fe06d210563188052d08dab146a8463fdb6898d655", - "f12ff4c12722f2c901885c67d232c325b604d54e5b67c35da01ab133fd36e637bf8d2501b463ffb6e4438efaf2a59526a85218e00c0f6b7b5594c8f4154c1ece", + "027346e0349f948c0a2e7e9badb67d27fcc8ff4d5eacff1e5dd6a09c23a54d6793bf7ef1f25c9ed6b8c74f49d86d7b87478b7a00e24ea72e2ed2cadc0286c761", + "027346e0349f948c0a2e7e9badb67d27fcc8ff4d5eacff1e5dd6a09c23a54d6793bf7ef1f25c9ed6b8c74f49d86d7b87478b7a00e24ea72e2ed2cadc0286c761", + "1a2b32fe2b252ec622e5a15af21b274d702faa623d09c6fc51a44e7562cc84ac8b8c368d98d284dfb6666680ee252b071d5fbff44564a952ebaa12fe6f389e68", None )] #[test_case( diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index 9b26069e0adad..ba2d9c040a680 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -66,7 +66,7 @@ impl HttpCache { Ok(()) } - async fn exists( + pub async fn exists( &self, hash: &str, token: &str, diff --git a/crates/turborepo-vercel-api-mock/Cargo.toml b/crates/turborepo-vercel-api-mock/Cargo.toml index bd36d9cc69ef3..a88d802a697f2 100644 --- a/crates/turborepo-vercel-api-mock/Cargo.toml +++ b/crates/turborepo-vercel-api-mock/Cargo.toml @@ -13,6 +13,6 @@ axum-macros = "0.3.7" axum-server = "0.4.7" futures-util = "0.3.28" port_scanner = { workspace = true } -tempfile.workspace = true +tempfile = { workspace = true } tokio = { workspace = true, features = ["full"] } turborepo-api-client = { workspace = true } From f05f324dd1420933786a255b8892348782422089 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Mon, 10 Jul 2023 16:04:29 -0400 Subject: [PATCH 10/14] Now returning a CacheResponse, the Rust equivalent of ItemStatus --- Cargo.lock | 1 - crates/turborepo-cache/src/http.rs | 32 ++++++++++++++++++++---------- crates/turborepo-cache/src/lib.rs | 12 +++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad2016d6dfd67..07fad6bb119c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9591,7 +9591,6 @@ dependencies = [ "path-clean 1.0.1", "petgraph", "port_scanner", - "ring", "serde", "serde_json", "sha2", diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index ba2d9c040a680..c063f658bbf3d 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -6,7 +6,7 @@ use turborepo_api_client::{APIClient, Response}; use crate::{ cache_archive::{CacheReader, CacheWriter}, signature_authentication::ArtifactSignatureAuthenticator, - CacheError, + CacheError, CacheResponse, CacheSource, }; pub struct HttpCache { @@ -73,7 +73,7 @@ impl HttpCache { team_id: &str, team_slug: Option<&str>, use_preflight: bool, - ) -> Result { + ) -> Result { let response = self .client .artifact_exists(hash, token, team_id, team_slug, use_preflight) @@ -81,7 +81,10 @@ impl HttpCache { let duration = Self::get_duration_from_response(&response)?; - Ok(duration) + Ok(CacheResponse { + source: CacheSource::Remote, + time_saved: duration, + }) } fn get_duration_from_response(response: &Response) -> Result { @@ -105,7 +108,7 @@ impl HttpCache { team_id: &str, team_slug: Option<&str>, use_preflight: bool, - ) -> Result<(Vec, u32), CacheError> { + ) -> Result<(CacheResponse, Vec), CacheError> { let response = self .client .fetch_artifact(hash, token, team_id, team_slug, use_preflight) @@ -148,7 +151,13 @@ impl HttpCache { let files = Self::restore_tar(&self.repo_root, &body)?; - Ok((files, duration)) + Ok(( + CacheResponse { + source: CacheSource::Remote, + time_saved: duration, + }, + files, + )) } pub(crate) fn restore_tar( @@ -169,7 +178,7 @@ mod test { use turborepo_api_client::APIClient; use vercel_api_mock::start_test_server; - use crate::http::HttpCache; + use crate::{http::HttpCache, CacheSource}; struct TestFile { path: AnchoredSystemPathBuf, @@ -234,12 +243,13 @@ mod test { ) .await?; - let received_duration = cache.exists(hash, "", "", None, false).await; - assert!(received_duration.is_ok()); - assert_eq!(received_duration.unwrap(), duration); + let cache_response = cache.exists(hash, "", "", None, false).await?; - let (received_files, received_duration) = cache.retrieve(hash, "", "", None, false).await?; - assert_eq!(received_duration, duration); + assert_eq!(cache_response.time_saved, duration); + assert_eq!(cache_response.source, CacheSource::Remote); + + let (cache_response, received_files) = cache.retrieve(hash, "", "", None, false).await?; + assert_eq!(cache_response.time_saved, duration); for (test_file, received_file) in files.iter().zip(received_files) { assert_eq!(received_file, test_file.path); diff --git a/crates/turborepo-cache/src/lib.rs b/crates/turborepo-cache/src/lib.rs index 60d67c717858d..2d5b5d77b1c51 100644 --- a/crates/turborepo-cache/src/lib.rs +++ b/crates/turborepo-cache/src/lib.rs @@ -51,3 +51,15 @@ pub enum CacheError { #[error("tar attempts to write outside of directory: {0}")] LinkOutsideOfDirectory(String, #[backtrace] Backtrace), } + +#[derive(Debug, Clone, PartialEq)] +pub enum CacheSource { + Local, + Remote, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CacheResponse { + source: CacheSource, + time_saved: u32, +} From 39857b344e79875b03ffa43710a53408f0d4422d Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Tue, 11 Jul 2023 13:32:49 -0400 Subject: [PATCH 11/14] PR feedback --- crates/turborepo-api-client/src/lib.rs | 10 +++--- crates/turborepo-cache/src/http.rs | 4 +-- crates/turborepo-vercel-api-mock/src/lib.rs | 35 ++++++++++++++------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index 795369afe7d99..947949db8fd54 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -122,7 +122,7 @@ pub struct UserResponse { pub struct PreflightResponse { location: Url, - allow_auth: bool, + allow_authorization_header: bool, } pub struct APIClient { @@ -272,11 +272,11 @@ impl APIClient { token, &request_url, "PUT", - "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag", + "Authorization, Content-Type, User-Agent, x-artifact-duration, x-artifact-tag", ) .await?; - allow_auth = preflight_response.allow_auth; + allow_auth = preflight_response.allow_authorization_header; request_url = preflight_response.location.to_string(); } @@ -346,7 +346,7 @@ impl APIClient { .do_preflight(token, &request_url, "GET", "Authorization, User-Agent") .await?; - allow_auth = preflight_response.allow_auth; + allow_auth = preflight_response.allow_authorization_header; request_url = preflight_response.location.to_string(); }; @@ -401,7 +401,7 @@ impl APIClient { Ok(PreflightResponse { location, - allow_auth, + allow_authorization_header: allow_auth, }) } diff --git a/crates/turborepo-cache/src/http.rs b/crates/turborepo-cache/src/http.rs index c063f658bbf3d..89a6f8d72500c 100644 --- a/crates/turborepo-cache/src/http.rs +++ b/crates/turborepo-cache/src/http.rs @@ -88,8 +88,8 @@ impl HttpCache { } fn get_duration_from_response(response: &Response) -> Result { - if let Some(duration) = response.headers().get("x-artifact-duration") { - let duration = duration + if let Some(duration_value) = response.headers().get("x-artifact-duration") { + let duration = duration_value .to_str() .map_err(|_| CacheError::InvalidDuration(Backtrace::capture()))?; diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index 7fa6d8e47fd13..137220bc74691 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -32,11 +32,12 @@ pub const EXPECTED_SSO_TEAM_ID: &str = "expected_sso_team_id"; pub const EXPECTED_SSO_TEAM_SLUG: &str = "expected_sso_team_slug"; pub async fn start_test_server(port: u16) -> Result<()> { - let durations = Arc::new(Mutex::new(HashMap::new())); - let durations2 = durations.clone(); - let durations3 = durations.clone(); - let tempdir = Arc::new(tempfile::tempdir()?); - let tempdir2 = tempdir.clone(); + let get_durations_ref = Arc::new(Mutex::new(HashMap::new())); + let head_durations_ref = get_durations_ref.clone(); + let put_durations_ref = get_durations_ref.clone(); + let put_tempdir_ref = Arc::new(tempfile::tempdir()?); + let get_tempdir_ref = put_tempdir_ref.clone(); + let app = Router::new() .route( "/v2/user", @@ -99,7 +100,7 @@ pub async fn start_test_server(port: u16) -> Result<()> { "/v8/artifacts/:hash", put( |Path(hash): Path, headers: HeaderMap, mut body: BodyStream| async move { - let root_path = tempdir.path(); + let root_path = put_tempdir_ref.path(); let file_path = root_path.join(&hash); let mut file = OpenOptions::new() .append(true) @@ -110,10 +111,10 @@ pub async fn start_test_server(port: u16) -> Result<()> { let duration = headers .get("x-artifact-duration") .and_then(|header_value| header_value.to_str().ok()) - .and_then(|duration| duration.parse::().ok()) - .unwrap_or(0); + .and_then(|duration| duration.parse::().ok()) + .expect("x-artifact-duration header is missing"); - let mut durations_map = durations.lock().await; + let mut durations_map = put_durations_ref.lock().await; durations_map.insert(hash.clone(), duration); while let Some(item) = body.next().await { @@ -128,10 +129,15 @@ pub async fn start_test_server(port: u16) -> Result<()> { .route( "/v8/artifacts/:hash", get(|Path(hash): Path| async move { - let root_path = tempdir2.path(); + let root_path = get_tempdir_ref.path(); let file_path = root_path.join(&hash); let buffer = std::fs::read(file_path).unwrap(); - let duration = durations2.lock().await.get(&hash).cloned().unwrap_or(0); + let duration = get_durations_ref + .lock() + .await + .get(&hash) + .cloned() + .unwrap_or(0); let mut headers = HeaderMap::new(); headers.insert( @@ -145,7 +151,12 @@ pub async fn start_test_server(port: u16) -> Result<()> { .route( "/v8/artifacts/:hash", head(|Path(hash): Path| async move { - let duration = durations3.lock().await.get(&hash).cloned().unwrap_or(0); + let duration = head_durations_ref + .lock() + .await + .get(&hash) + .cloned() + .unwrap_or(0); let mut headers = HeaderMap::new(); headers.insert( From 637c4d336de89d029b1cf29e6ddbb7351bcf2ded Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Wed, 12 Jul 2023 11:49:25 -0400 Subject: [PATCH 12/14] Added test for relative location header --- Cargo.lock | 2 + crates/turborepo-api-client/Cargo.toml | 6 ++- crates/turborepo-api-client/src/lib.rs | 52 ++++++++++++++++++++- crates/turborepo-vercel-api-mock/src/lib.rs | 20 +++++++- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07fad6bb119c5..41cff065e6956 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9565,12 +9565,14 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", + "port_scanner", "reqwest", "rustc_version_runtime", "serde", "thiserror", "tokio", "url", + "vercel-api-mock", ] [[package]] diff --git a/crates/turborepo-api-client/Cargo.toml b/crates/turborepo-api-client/Cargo.toml index 1c324109661f0..a986b0cac3224 100644 --- a/crates/turborepo-api-client/Cargo.toml +++ b/crates/turborepo-api-client/Cargo.toml @@ -9,6 +9,10 @@ license = "MPL-2.0" native-tls = ["reqwest/native-tls"] rustls-tls = ["reqwest/rustls-tls"] +[dev-dependencies] +port_scanner = { workspace = true } +vercel-api-mock = { workspace = true } + [dependencies] anyhow = { workspace = true } chrono = { workspace = true, features = ["serde"] } @@ -16,5 +20,5 @@ reqwest = { workspace = true, features = ["json"] } rustc_version_runtime = "0.2.1" serde = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["full"] } url = { workspace = true } diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index 947949db8fd54..ee7354d43e8e9 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -388,7 +388,14 @@ impl APIClient { let headers = response.headers(); let location = if let Some(location) = headers.get("Location") { let location = location.to_str()?; - Url::parse(location)? + + match Url::parse(location) { + Ok(location_url) => location_url, + Err(url::ParseError::RelativeUrlWithoutBase) => { + Url::parse(&self.base_url)?.join(location)? + } + Err(e) => return Err(e.into()), + } } else { response.url().clone() }; @@ -438,3 +445,46 @@ impl APIClient { format!("{}{}", self.base_url, endpoint) } } + +#[cfg(test)] +mod test { + use anyhow::Result; + use url::Url; + use vercel_api_mock::start_test_server; + + use crate::APIClient; + + #[tokio::test] + async fn test_do_preflight() -> Result<()> { + let port = port_scanner::request_open_port().unwrap(); + let handle = tokio::spawn(start_test_server(port)); + let base_url = format!("http://localhost:{}", port); + + let client = APIClient::new(&base_url, 200, "2.0.0", true)?; + + let response = client + .do_preflight( + "", + &format!("{}/preflight/absolute-location", base_url), + "GET", + "Authorization, User-Agent", + ) + .await; + + assert_eq!(response.is_ok()); + + let response = client + .do_preflight( + "", + &format!("{}/preflight/relative-location", base_url), + "GET", + "Authorization, User-Agent", + ) + .await; + + assert_eq!(response.is_ok()); + + handle.abort(); + Ok(()) + } +} diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index 137220bc74691..963bb0760f2ac 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -4,7 +4,7 @@ use anyhow::Result; use axum::{ extract::{BodyStream, Path}, http::{HeaderMap, HeaderValue, StatusCode}, - routing::{get, head, put}, + routing::{get, head, options, put}, Json, Router, }; use futures_util::StreamExt; @@ -164,6 +164,24 @@ pub async fn start_test_server(port: u16) -> Result<()> { HeaderValue::from_str(&duration.to_string()).unwrap(), ); + headers + }), + ) + .route( + "/preflight/absolute-location", + options(|| async { + let mut headers = HeaderMap::new(); + headers.insert("Location", "http://example.com/about".parse().unwrap()); + + headers + }), + ) + .route( + "/preflight/relative-location", + options(|| async { + let mut headers = HeaderMap::new(); + headers.insert("Location", "/about/me".parse().unwrap()); + headers }), ); From db180aab8d2f13090d6e6110dfe9afe37aa870d5 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Wed, 12 Jul 2023 12:15:44 -0400 Subject: [PATCH 13/14] Added regex for authorization and tests --- Cargo.lock | 96 +++++++++++---------- Cargo.toml | 2 +- crates/turborepo-api-client/Cargo.toml | 2 + crates/turborepo-api-client/src/lib.rs | 38 +++++++- crates/turborepo-vercel-api-mock/src/lib.rs | 26 ++++++ 5 files changed, 112 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41cff065e6956..17bc9f408ef5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,7 +206,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -326,7 +326,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -392,7 +392,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -558,7 +558,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -675,7 +675,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.25", + "syn 2.0.16", "which", ] @@ -1616,20 +1616,20 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 4.1.11", "criterion-plot", "futures", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -1793,7 +1793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b" dependencies = [ "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -1867,7 +1867,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -1884,7 +1884,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -1931,7 +1931,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -1953,7 +1953,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -2286,7 +2286,7 @@ dependencies = [ "darling 0.20.1", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -2512,7 +2512,7 @@ dependencies = [ "pmutil", "proc-macro2", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -2648,7 +2648,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -3447,7 +3447,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -4093,7 +4093,7 @@ checksum = "4c65c625186a9bcce6699394bee511e1b1aec689aa7e3be1bf4e996e75834153" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -4982,7 +4982,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 1.0.109", ] [[package]] @@ -5063,7 +5063,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -5223,7 +5223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -6092,9 +6092,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.171" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] @@ -6131,13 +6131,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -6577,7 +6577,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -6683,7 +6683,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -6939,7 +6939,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7027,7 +7027,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7169,7 +7169,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7325,7 +7325,7 @@ dependencies = [ "swc_ecma_ast", "swc_ecma_parser", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7436,7 +7436,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7659,7 +7659,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7709,7 +7709,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7818,7 +7818,7 @@ checksum = "ff9719b6085dd2824fd61938a881937be14b08f95e2d27c64c825a9f65e052ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7842,7 +7842,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -7858,9 +7858,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.25" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ "proc-macro2", "quote", @@ -8106,7 +8106,7 @@ dependencies = [ "quote", "regex", "relative-path", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -8158,7 +8158,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -8340,7 +8340,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.16", ] [[package]] @@ -8568,7 +8568,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 1.0.109", ] [[package]] @@ -9565,7 +9565,9 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", + "lazy_static", "port_scanner", + "regex", "reqwest", "rustc_version_runtime", "serde", @@ -9780,8 +9782,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.4.6", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index 4ab45d135c59d..6d9ff6a0e9ee9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ clap_complete = "4.1.2" concurrent-queue = "2.1.0" console = "0.15.5" console-subscriber = "0.1.8" -criterion = "0.4.0" +criterion = "0.5.1" crossbeam-channel = "0.5.8" dashmap = "5.4.0" dialoguer = "0.10.3" diff --git a/crates/turborepo-api-client/Cargo.toml b/crates/turborepo-api-client/Cargo.toml index a986b0cac3224..23ebf7caee167 100644 --- a/crates/turborepo-api-client/Cargo.toml +++ b/crates/turborepo-api-client/Cargo.toml @@ -16,6 +16,8 @@ vercel-api-mock = { workspace = true } [dependencies] anyhow = { workspace = true } chrono = { workspace = true, features = ["serde"] } +lazy_static = { workspace = true } +regex = { workspace = true } reqwest = { workspace = true, features = ["json"] } rustc_version_runtime = "0.2.1" serde = { workspace = true } diff --git a/crates/turborepo-api-client/src/lib.rs b/crates/turborepo-api-client/src/lib.rs index ee7354d43e8e9..bbceabac01c29 100644 --- a/crates/turborepo-api-client/src/lib.rs +++ b/crates/turborepo-api-client/src/lib.rs @@ -4,6 +4,8 @@ use std::env; +use lazy_static::lazy_static; +use regex::Regex; pub use reqwest::Response; use reqwest::{Method, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -14,6 +16,11 @@ pub use crate::error::{Error, Result}; mod error; mod retry; +lazy_static! { + static ref AUTHORIZATION_REGEX: Regex = + Regex::new(r"(?i)(?:^|,) *authorization *(?:,|$)").unwrap(); +} + #[derive(Debug, Clone, Deserialize)] pub struct VerifiedSsoUser { pub token: String, @@ -404,7 +411,7 @@ impl APIClient { .get("Access-Control-Allow-Headers") .map_or("", |h| h.to_str().unwrap_or("")); - let allow_auth = allowed_headers.to_lowercase().contains("authorization"); + let allow_auth = AUTHORIZATION_REGEX.is_match(allowed_headers); Ok(PreflightResponse { location, @@ -449,7 +456,6 @@ impl APIClient { #[cfg(test)] mod test { use anyhow::Result; - use url::Url; use vercel_api_mock::start_test_server; use crate::APIClient; @@ -471,7 +477,7 @@ mod test { ) .await; - assert_eq!(response.is_ok()); + assert!(response.is_ok()); let response = client .do_preflight( @@ -482,7 +488,31 @@ mod test { ) .await; - assert_eq!(response.is_ok()); + // Since PreflightResponse returns a Url, + // do_preflight would error if the Url is relative + assert!(response.is_ok()); + + let response = client + .do_preflight( + "", + &format!("{}/preflight/allow-auth", base_url), + "GET", + "Authorization, User-Agent", + ) + .await?; + + assert!(response.allow_authorization_header); + + let response = client + .do_preflight( + "", + &format!("{}/preflight/no-allow-auth", base_url), + "GET", + "Authorization, User-Agent", + ) + .await?; + + assert!(!response.allow_authorization_header); handle.abort(); Ok(()) diff --git a/crates/turborepo-vercel-api-mock/src/lib.rs b/crates/turborepo-vercel-api-mock/src/lib.rs index 963bb0760f2ac..a859ff700eec2 100644 --- a/crates/turborepo-vercel-api-mock/src/lib.rs +++ b/crates/turborepo-vercel-api-mock/src/lib.rs @@ -182,6 +182,32 @@ pub async fn start_test_server(port: u16) -> Result<()> { let mut headers = HeaderMap::new(); headers.insert("Location", "/about/me".parse().unwrap()); + headers + }), + ) + .route( + "/preflight/allow-auth", + options(|| async { + let mut headers = HeaderMap::new(); + headers.insert( + "Access-Control-Allow-Headers", + "Authorization, Location, Access-Control-Allow-Headers" + .parse() + .unwrap(), + ); + + headers + }), + ) + .route( + "/preflight/no-allow-auth", + options(|| async { + let mut headers = HeaderMap::new(); + headers.insert( + "Access-Control-Allow-Headers", + "x-authorization-foo, Location".parse().unwrap(), + ); + headers }), ); From fbaca53bafb20c468d7daea84e8bcb4805b851c6 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Wed, 12 Jul 2023 12:20:10 -0400 Subject: [PATCH 14/14] Reverting Cargo.toml changes --- Cargo.lock | 187 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 92 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17bc9f408ef5c..bade5c59c66e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,7 +206,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -326,7 +326,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -392,7 +392,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -558,7 +558,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -675,7 +675,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.16", + "syn 2.0.25", "which", ] @@ -741,9 +741,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.0" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" +checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" dependencies = [ "arrayref", "arrayvec 0.7.2", @@ -1401,9 +1401,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" +checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" [[package]] name = "convert_case" @@ -1616,20 +1616,20 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" dependencies = [ "anes", + "atty", "cast", "ciborium", - "clap 4.1.11", + "clap 3.2.23", "criterion-plot", "futures", - "is-terminal", "itertools", + "lazy_static", "num-traits", - "once_cell", "oorandom", "plotters", "rayon", @@ -1793,7 +1793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b" dependencies = [ "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -1867,7 +1867,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -1884,7 +1884,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -1931,7 +1931,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -1953,7 +1953,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -1971,9 +1971,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "dav1d" @@ -2061,9 +2061,9 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -2286,7 +2286,7 @@ dependencies = [ "darling 0.20.1", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -2512,7 +2512,7 @@ dependencies = [ "pmutil", "proc-macro2", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -2648,7 +2648,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -3413,9 +3413,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -3447,7 +3447,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -3870,9 +3870,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -3886,11 +3886,10 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" dependencies = [ - "cfg-if 1.0.0", "value-bag", ] @@ -4093,7 +4092,7 @@ checksum = "4c65c625186a9bcce6699394bee511e1b1aec689aa7e3be1bf4e996e75834153" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -4107,9 +4106,9 @@ dependencies = [ [[package]] name = "mimalloc-rust" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6973866e0bc6504c03a16b6817b7e70839cc8a1dbd5d6dab00c65d8034868d8b" +checksum = "5eb726c8298efb4010b2c46d8050e4be36cf807b9d9e98cb112f830914fc9bbe" dependencies = [ "cty", "mimalloc-rust-sys", @@ -4117,9 +4116,9 @@ dependencies = [ [[package]] name = "mimalloc-rust-sys" -version = "1.7.6-source" +version = "1.7.9-source" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a50daf45336b979a202a19f53b4b382f2c4bd50f392a8dbdb4c6c56ba5dfa64" +checksum = "6413e13241a9809f291568133eca6694572cf528c1a6175502d090adce5dd5db" dependencies = [ "cc", "cty", @@ -4967,22 +4966,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.25", ] [[package]] @@ -5063,7 +5062,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -5223,7 +5222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -5396,9 +5395,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -5617,13 +5616,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6cf59af1067a3fb53fbe5c88c053764e930f932be1d71d3ffe032cbe147f59" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick 1.0.1", "memchr", - "regex-syntax 0.7.0", + "regex-syntax 0.7.2", ] [[package]] @@ -5643,9 +5642,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "region" @@ -5916,9 +5915,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.13" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags 1.3.2", "errno", @@ -6092,9 +6091,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] @@ -6131,13 +6130,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -6577,7 +6576,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -6683,7 +6682,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -6725,9 +6724,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "supports-color" @@ -6939,7 +6938,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7027,7 +7026,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7169,7 +7168,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7325,7 +7324,7 @@ dependencies = [ "swc_ecma_ast", "swc_ecma_parser", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7436,7 +7435,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7659,7 +7658,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7709,7 +7708,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7818,7 +7817,7 @@ checksum = "ff9719b6085dd2824fd61938a881937be14b08f95e2d27c64c825a9f65e052ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7842,7 +7841,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -7858,9 +7857,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.16" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -8106,7 +8105,7 @@ dependencies = [ "quote", "regex", "relative-path", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -8158,7 +8157,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -8304,9 +8303,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -8319,7 +8318,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -8334,13 +8333,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.25", ] [[package]] @@ -8562,13 +8561,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.25", ] [[package]] @@ -8584,9 +8583,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -9888,9 +9887,9 @@ checksum = "d70b6494226b36008c8366c288d77190b3fad2eb4c10533139c1c1f461127f1a" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-linebreak" @@ -10011,13 +10010,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.0.0-alpha.9" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" -dependencies = [ - "ctor 0.1.26", - "version_check", -] +checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" [[package]] name = "vcpkg" diff --git a/Cargo.toml b/Cargo.toml index 6d9ff6a0e9ee9..4ab45d135c59d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ clap_complete = "4.1.2" concurrent-queue = "2.1.0" console = "0.15.5" console-subscriber = "0.1.8" -criterion = "0.5.1" +criterion = "0.4.0" crossbeam-channel = "0.5.8" dashmap = "5.4.0" dialoguer = "0.10.3"