refactor: Migrate gha cache to opendal based (#1528)

Signed-off-by: Xuanwo <github@xuanwo.io>

.github/workflows/integration-tests.yml

@@ -189,3 +189,52 @@ jobs:
           ${SCCACHE_PATH} --show-stats
 
           ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"
+
+  gha:
+    runs-on: ubuntu-latest
+    needs: build
+
+    env:
+      SCCACHE_GHA_VERSION: "sccache_integration_tests"
+      RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+
+      - name: Configure Cache Env
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Install rust
+        uses: ./.github/actions/rust-toolchain
+        with:
+          toolchain: "stable"
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: integration-tests
+          path: /home/runner/.cargo/bin/
+      - name: Chmod for binary
+        run: chmod +x ${SCCACHE_PATH}
+
+      - name: Test
+        run: cargo clean && cargo build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+          ${SCCACHE_PATH} --show-stats | grep gha
+
+      - name: Test Twice for Cache Read
+        run: cargo clean && cargo build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+          ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"

Cargo.toml

@@ -28,7 +28,7 @@ bincode = "1"
 blake3 = "1"
 byteorder = "1.0"
 bytes = "1"
-opendal = { version= "0.22.6", optional=true }
+opendal = { version= "0.24", optional=true }
 reqsign = {version="0.7.3", optional=true}
 chrono = { version = "0.4.23", optional = true }
 clap = { version = "4.0.29", features = ["derive", "env", "wrap_help"] }
@@ -38,7 +38,6 @@ filetime = "0.2"
 flate2 = { version = "1.0", optional = true, default-features = false, features = ["rust_backend"] }
 futures = "0.3"
 futures-locks = "0.7"
-gha-toolkit = { version = "0.3.1", optional = true }
 gzp = { version = "0.11.1", default-features = false, features = ["deflate_rust"]  }
 http = "0.2"
 hyper = { version = "0.14.10", optional = true, features = ["server"] }
@@ -123,7 +122,7 @@ all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha"]
 azure = ["opendal","reqsign"]
 s3 = ["opendal","reqsign"]
 gcs = ["opendal","reqsign", "url", "reqwest/blocking"]
-gha = ["gha-toolkit"]
+gha = ["opendal"]
 memcached = ["memcached-rs"]
 native-zlib = []
 redis = ["url", "opendal/services-redis"]

docs/GHA.md

@@ -1,8 +1,8 @@
 # GitHub Actions
 
-To use the [GitHub Actions cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows), you need to set the `SCCACHE_GHA_CACHE_URL`/`ACTIONS_CACHE_URL` and `SCCACHE_GHA_RUNTIME_TOKEN`/`ACTIONS_RUNTIME_TOKEN` environmental variables. The `SCCACHE_` prefixed environmental variables override the variables without the prefix.
+To use the [GitHub Actions cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows), you need to set the `SCCACHE_GHA_VERSION` which is a namespace for the whole cache set.
 
-In a GitHub Actions workflow, you can set these environmental variables using the following step.
+This cache type will needs token like `ACTIONS_CACHE_URL` and `ACTIONS_RUNTIME_TOKEN` to work. You can set these environmental variables using the following step in a GitHub Actions workflow.
 
 ```yaml
 - name: Configure sccache
@@ -12,17 +12,3 @@ In a GitHub Actions workflow, you can set these environmental variables using th
       core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
       core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
 ```
-
-To write to the cache, set `SCCACHE_GHA_CACHE_TO` to a cache key, for example
-`sccache-latest`. To read from cache key prefixes, set `SCCACHE_GHA_CACHE_FROM`
-to a comma-separated list of cache key prefixes, for example `sccache-`.
-
-In contrast to the [`@actions/cache`](https://github.com/actions/cache) action, which saves a single large archive per cache key, `sccache` with GHA cache storage saves each cache entry separately.
-
-GHA cache storage will create many small caches with the same cache key, e.g. `SCCACHE_GHA_CACHE_TO` and `SCCACHE_GHA_CACHE_FROM`. These GHA caches are differentiated by their [_version_](https://github.com/actions/cache#cache-version). The GHA cache implementation in `sccache` calculates the cache version from the [`sccache` entry key](docs/Caching.md), e.g. the source file path.
-
-For example, if a cache entry has the version `main.rs` and has GHA cache entries for the `sccache-1` and `sccache-2` keys, then `SCCACHE_GHA_CACHE_FROM=sccache-` will match both and [return the most recent entry](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key).
-
-This behavior is useful for scoping caches from different versions of Rust or for cross-platform builds (`rust-sdk-{RUST_TOOLKIT}-{TARGET_TRIPLE}-`), and to allow newer commits to override older caches by adding the Git SHA as a suffix (`-{GITHUB_SHA}`), as in the following screenshot.
-
-<img width="718" src="https://user-images.githubusercontent.com/19253212/205356799-deedc465-e534-4ef6-a249-fc15121fdfd9.png">

src/cache/cache.rs

@@ -394,6 +394,7 @@ impl Storage for opendal::Operator {
 
         let can_write = match self.object(path).write("Hello, World!").await {
             Ok(_) => true,
+            Err(err) if err.kind() == ErrorKind::ObjectAlreadyExists => true,
             Err(err) if err.kind() == ErrorKind::ObjectPermissionDenied => false,
             Err(err) => bail!("cache storage failed to write: {:?}", err),
         };
@@ -485,15 +486,10 @@ pub fn storage_from_config(
                 return Ok(Arc::new(storage));
             }
             #[cfg(feature = "gha")]
-            CacheType::GHA(config::GHACacheConfig {
-                ref url,
-                ref token,
-                ref cache_to,
-                ref cache_from,
-            }) => {
-                debug!("Init gha cache with url {url}");
+            CacheType::GHA(config::GHACacheConfig { ref version }) => {
+                debug!("Init gha cache with version {version}");
 
-                let storage = GHACache::new(url, token, cache_to.clone(), cache_from.clone())
+                let storage = GHACache::build(version)
                     .map_err(|err| anyhow!("create gha cache failed: {err:?}"))?;
                 return Ok(Arc::new(storage));
             }