feat: Add SCCACHE_GCS_CREDENTIALS_URL feature back for gcs

Cargo.toml

@@ -28,7 +28,8 @@ bincode = "1"
 blake3 = "1"
 byteorder = "1.0"
 bytes = "1"
-opendal = { version= "0.22.5", optional=true }
+opendal = { version= "0.22.6", optional=true }
+reqsign = {version="0.7.3", optional=true}
 chrono = { version = "0.4.23", optional = true }
 clap = { version = "4.0.29", features = ["derive", "env", "wrap_help"] }
 directories = "4.0.1"
@@ -120,9 +121,9 @@ features = [
 [features]
 default = ["all"]
 all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha"]
-azure = ["opendal"]
-s3 = ["opendal"]
-gcs = ["opendal"]
+azure = ["opendal","reqsign"]
+s3 = ["opendal","reqsign"]
+gcs = ["opendal","reqsign"]
 gha = ["gha-toolkit"]
 memcached = ["memcached-rs"]
 native-zlib = []

docs/Gcs.md

@@ -25,6 +25,7 @@ Cache location                  GCS, bucket: Bucket(name=<bucket name in GCP>),
 Sccache is able to load credentials from various sources. Including:
 
 - User Input: If `SCCACHE_GCS_KEY_PATH` has been set, we will load from key path first.
+- [Task Cluster](https://taskcluster.net/): If `SCCACHE_GCS_CREDENTIALS_URL` has been set, we will load token from this url first.
 - Static: `GOOGLE_APPLICATION_CREDENTIALS`
 - Well-known locations:
   - Windows: `%APPDATA%\gcloud\application_default_credentials.json`
@@ -35,4 +36,4 @@ Sccache is able to load credentials from various sources. Including:
 
 ## Deprecation
 
-`SCCACHE_GCS_CREDENTIALS_URL` and `SCCACHE_GCS_OAUTH_URL` have been deprecated and not supported, please use `SCCACHE_GCS_SERVICE_ACCOUNT` instead.
+`SCCACHE_GCS_OAUTH_URL` have been deprecated and not supported, please use `SCCACHE_GCS_SERVICE_ACCOUNT` instead.

src/cache/cache.rs

@@ -402,6 +402,7 @@ pub fn storage_from_config(config: &Config, pool: &tokio::runtime::Handle) -> Ar
                 ref cred_path,
                 rw_mode,
                 ref service_account,
+                ref credential_url,
             }) => {
                 debug!(
                     "Trying GCS bucket({}, {}, {:?})",
@@ -420,6 +421,7 @@ pub fn storage_from_config(config: &Config, pool: &tokio::runtime::Handle) -> Ar
                         cred_path.as_deref(),
                         service_account.as_deref(),
                         gcs_read_write_mode,
+                        credential_url.as_deref(),
                     ) {
                         Ok(s) => {
                             trace!("Using GCSCache");

src/cache/gcs.rs

@@ -16,6 +16,7 @@
 use crate::errors::*;
 use opendal::services::gcs;
 use opendal::Operator;
+use reqsign::{GoogleBuilder, GoogleToken, GoogleTokenLoad};
 
 #[derive(Copy, Clone)]
 pub enum RWMode {
@@ -43,18 +44,75 @@ impl GCSCache {
         cred_path: Option<&str>,
         service_account: Option<&str>,
         rw_mode: RWMode,
+        credential_url: Option<&str>,
     ) -> Result<Operator> {
         let mut builder = gcs::Builder::default();
         builder.bucket(bucket);
         builder.root(key_prefix);
-        builder.scope(rw_mode.to_scope());
+
+        let mut signer_builder = GoogleBuilder::default();
+        signer_builder.scope(rw_mode.to_scope());
         if let Some(service_account) = service_account {
-            builder.service_account(service_account);
+            signer_builder.service_account(service_account);
         }
         if let Some(path) = cred_path {
-            builder.credential_path(path);
+            signer_builder.credential_path(path);
+        }
+        if let Some(cred_url) = credential_url {
+            signer_builder.customed_token_loader(TaskClusterTokenLoader {
+                client: reqwest::blocking::Client::default(),
+                scope: rw_mode.to_scope().to_string(),
+                url: cred_url.to_string(),
+            });
         }
+        builder.signer(signer_builder.build()?);
 
         Ok(builder.build()?.into())
     }
 }
+
+/// TaskClusterTokenLoeader is used to load tokens from [TaskCluster](https://taskcluster.net/)
+///
+/// This feature is required to run [mozilla's CI](https://searchfox.org/mozilla-central/source/build/mozconfig.cache#67-84):
+///
+/// ```txt
+/// export SCCACHE_GCS_CREDENTIALS_URL=http://taskcluster/auth/v1/gcp/credentials/$SCCACHE_GCS_PROJECT/${bucket}@$SCCACHE_GCS_PROJECT.iam.gserviceaccount.com"
+/// ```
+///
+/// Reference: [gcpCredentials](https://docs.taskcluster.net/docs/reference/platform/auth/api#gcpCredentials)
+#[derive(Debug)]
+struct TaskClusterTokenLoader {
+    client: reqwest::blocking::Client,
+    scope: String,
+    url: String,
+}
+
+impl GoogleTokenLoad for TaskClusterTokenLoader {
+    fn load_token(&self) -> Result<Option<GoogleToken>> {
+        let res = self.client.get(&self.url).send()?;
+
+        if res.status().is_success() {
+            let resp = res.json::<TaskClusterToken>()?;
+
+            // TODO: we can parse expire time instead using hardcode 1 hour.
+            Ok(Some(GoogleToken::new(
+                &resp.access_token,
+                3600,
+                &self.scope,
+            )))
+        } else {
+            let status_code = res.status();
+            let content = res.text()?;
+            Err(anyhow!(
+                "token load failed for: code: {status_code}, {content}"
+            ))
+        }
+    }
+}
+
+#[derive(Deserialize, Default)]
+#[serde(default, rename_all(deserialize = "camelCase"))]
+struct TaskClusterToken {
+    access_token: String,
+    expire_time: String,
+}

src/config.rs

@@ -186,6 +186,7 @@ pub struct GCSCacheConfig {
     pub cred_path: Option<String>,
     pub service_account: Option<String>,
     pub rw_mode: GCSCacheRWMode,
+    pub credential_url: Option<String>,
 }
 
 #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -535,15 +536,15 @@ fn config_from_env() -> Result<EnvConfig> {
             .unwrap_or_default()
             .to_owned();
 
-        if env::var("SCCACHE_GCS_CREDENTIALS_URL").is_ok() {
-            warn!("gcs deprecated_url has been deprecated");
-        }
+
 
         if env::var("SCCACHE_GCS_OAUTH_URL").is_ok() {
             warn!("SCCACHE_GCS_OAUTH_URL has been deprecated");
             warn!("if you intend to use vm metadata for auth, please set correct service account intead");
         }
 
+        let credential_url = env::var("SCCACHE_GCS_CREDENTIALS_URL").ok();
+
         let cred_path = env::var("SCCACHE_GCS_KEY_PATH").ok();
         let service_account = env::var("SCCACHE_GCS_SERVICE_ACCOUNT").ok();
 
@@ -568,6 +569,7 @@ fn config_from_env() -> Result<EnvConfig> {
             cred_path,
             service_account,
             rw_mode,
+            credential_url,
         }
     });
 
@@ -879,7 +881,7 @@ pub mod server {
 
     fn default_pot_clone_args() -> Vec<String> {
         DEFAULT_POT_CLONE_ARGS
-            .into_iter()
+            .iter()
             .map(|s| s.to_string())
             .collect()
     }
@@ -1128,6 +1130,7 @@ no_credentials = true
                     service_account: Some("example_service_account".to_string()),
                     rw_mode: GCSCacheRWMode::ReadOnly,
                     key_prefix: "prefix".into(),
+                    credential_url: None,
                 }),
                 gha: Some(GHACacheConfig {
                     url: "http://localhost".to_owned(),