From f615ebb3c39755d7c69d7b2dc2274b53deaf66b1 Mon Sep 17 00:00:00 2001
From: Kalaiselvi Murugesan <kalaiselvim@google.com>
Date: Wed, 29 Nov 2023 00:21:46 +0000
Subject: [PATCH 1/5] ci:save_cache and restore_cache

---
 cache/Dockerfile-base    |  13 ++++
 cache/Dockerfile-restore |   4 ++
 cache/Dockerfile-save    |   4 ++
 cache/checksum           |   3 +
 cache/cloudbuild.yaml    | 133 +++++++++++++++++++++++++++++++++++++++
 cache/restore_cache      | 108 +++++++++++++++++++++++++++++++
 cache/save_cache         |  92 +++++++++++++++++++++++++++
 7 files changed, 357 insertions(+)
 create mode 100644 cache/Dockerfile-base
 create mode 100644 cache/Dockerfile-restore
 create mode 100644 cache/Dockerfile-save
 create mode 100755 cache/checksum
 create mode 100644 cache/cloudbuild.yaml
 create mode 100755 cache/restore_cache
 create mode 100755 cache/save_cache

diff --git a/cache/Dockerfile-base b/cache/Dockerfile-base
new file mode 100644
index 0000000000..06228044d5
--- /dev/null
+++ b/cache/Dockerfile-base
@@ -0,0 +1,13 @@
+FROM gcr.io/cloud-builders/gcloud-slim
+
+RUN    apt-get update \
+    && apt-get install -y gcc python3-dev python3-setuptools python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install --no-cache-dir -U crcmod
+
+COPY checksum /bin
+COPY save_cache /bin
+COPY restore_cache /bin
+
+RUN chmod +x /bin/checksum /bin/save_cache /bin/restore_cache
\ No newline at end of file
diff --git a/cache/Dockerfile-restore b/cache/Dockerfile-restore
new file mode 100644
index 0000000000..7ffc911d53
--- /dev/null
+++ b/cache/Dockerfile-restore
@@ -0,0 +1,4 @@
+ARG project_id
+FROM us-docker.pkg.dev/${project_id}/ci/cache
+
+ENTRYPOINT ["restore_cache"]
\ No newline at end of file
diff --git a/cache/Dockerfile-save b/cache/Dockerfile-save
new file mode 100644
index 0000000000..4d62fa36ee
--- /dev/null
+++ b/cache/Dockerfile-save
@@ -0,0 +1,4 @@
+ARG project_id
+FROM us-docker.pkg.dev/${project_id}/ci/cache
+
+ENTRYPOINT ["save_cache"]
\ No newline at end of file
diff --git a/cache/checksum b/cache/checksum
new file mode 100755
index 0000000000..6ac8149d5d
--- /dev/null
+++ b/cache/checksum
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo $(cksum $@ | cut -d' ' -f1) | tr ' ' -
diff --git a/cache/cloudbuild.yaml b/cache/cloudbuild.yaml
new file mode 100644
index 0000000000..b6d235ba44
--- /dev/null
+++ b/cache/cloudbuild.yaml
@@ -0,0 +1,133 @@
+steps:
+
+- name: make-docker
+  id: build_base_image
+  args: 
+  - 'build'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+  - '--file=Dockerfile-base'
+  - '.'
+
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  id: build_save_cache
+  args: 
+  - 'build'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
+  - '--file=Dockerfile-save'
+  - '--build-arg=project_id=$PROJECT_ID'
+  - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+  - '.'
+
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+  id: build_restore_cache
+  args: 
+  - 'build'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
+  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'
+  - '--file=Dockerfile-restore'
+  - '--build-arg=project_id=$PROJECT_ID'
+  - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+  - '.'
+
+# Test the script
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: setup_test
+  entrypoint: 'bash'
+  args: 
+  - '-c'
+  - |
+    echo "Creating test cache file structure."
+    mkdir -p /original/folder1 /original/folder2/subfolder3 rel_folder
+    touch /original/folder1/file1.txt
+    touch /original/folder1/file2.txt
+    touch /original/folder2/ignore.txt
+    touch /original/folder2/subfolder3/file1.txt
+    touch rel_folder/file3.txt
+  volumes:
+  - name: original
+    path: /original
+
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  id: save_cache
+  args:
+  - '--out=/cached'
+  - '--key=simple-key-$( checksum cloudbuild.yaml )'
+  - '--path=/original/folder1'
+  - '--path=/original/folder2/subfolder3'
+  - '--path=rel_folder'
+  volumes:
+  - name: original
+    path: /original
+  - name: cached
+    path: /cached
+
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  id: verify_cache
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    echo "Verifying cache file exists."
+
+    cache_file="/cached/simple-key-$( checksum cloudbuild.yaml ).tgz"
+    if [[ ! -f "${cache_file}" ]];then
+      echo "Missing cache file at ${cache_file}"
+      echo "Contents:"
+      echo "$(ls -al /cached)"
+      exit 1
+    fi
+
+    echo "Cache tests passed."
+  volumes:
+  - name: cached
+    path: /cached
+
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: clean_cache
+  entrypoint: bash
+  args:
+  - -c
+  - |
+    echo "Clearing original files..."
+    rm -rf /original/*
+    rm -rf rel_folder/
+  volumes:
+  - name: original
+    path: /original
+
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+  id: restore_cache
+  args:
+  - '--src=/cached'
+  - '--key=simple-key-$( checksum cloudbuild.yaml )'
+  volumes:
+  - name: original
+    path: /original
+  - name: cached
+    path: /cached
+
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: verify_restore
+  entrypoint: bash
+  args:
+  - '-c'
+  - |
+    test -f /original/folder1/file1.txt
+    test -f /original/folder1/file2.txt
+    test -f /original/folder2/ignore.txt
+    test -f /original/folder2/subfolder3/file1.txt
+    test -f rel_folder/file3.txt
+    test -f /workspace/rel_folder/file3.txt
+  volumes:
+  - name: original
+    path: /original
+
+images: 
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
+- 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'
diff --git a/cache/restore_cache b/cache/restore_cache
new file mode 100755
index 0000000000..94dfb5a027
--- /dev/null
+++ b/cache/restore_cache
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+BUCKET=""
+SRC_DIR="."
+KEY="cache"
+KEY_FALLBACK_PATTERN=""
+PATHS=()
+
+function print_usage {
+  echo "Usage: $0 --bucket=gs://my-cache-bucket --key=cache-key"
+  echo
+  echo "Saves the specified paths to a cache file located in the out directory."
+  echo
+  echo "  -b,  --bucket       The cloud storage bucket to download the cache from. [optional]"
+  echo "  -s,  --src          The local directory in which the cache is stored. [optional]"
+  echo "  -k,  --key          The cache key used for this cache file. [optional]"
+  echo "  -kf, --key_fallback The cache key fallback pattern to be used if exact cache key is not found. It will try to find a file based on the pattern specified and will retrieve the most recent one. [optional]"
+  echo 
+}
+
+for i in "$@"; do
+  case $i in
+    -b=*|--bucket=*)
+      BUCKET="${i#*=}"
+      shift
+      ;;
+    -s=*|--src=* )
+      SRC_DIR="${i#*=}"
+      shift
+      ;;
+    -k=*|--key=* )
+      KEY="${i#*=}"
+      shift
+      ;;
+    -kf=*|--key_fallback=* )
+      KEY_FALLBACK_PATTERN="${i#*=}"
+      shift
+      ;;
+    -h|--help )
+      print_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: ${i}"
+      print_usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "${BUCKET}" ] && [ -z "${SRC_DIR}" ];then
+  echo "Require one of [--bucket, --src]"
+  print_usage
+  exit 1
+fi
+
+# Evaluate the key string so that bash variables work
+eval "KEY=\"$KEY\""
+eval "KEY_FALLBACK_PATTERN=\"$KEY_FALLBACK_PATTERN\""
+
+if [ -n "${BUCKET}" ];then
+  REMOTE_CACHE_FILE="${BUCKET}/${KEY}.tgz"
+
+  echo "Checking cache file existence for: ${REMOTE_CACHE_FILE}"
+  REMOTE_CACHE_FILE_EXISTENCE=$(gsutil -q stat ${REMOTE_CACHE_FILE}; echo $?)
+
+  # If cache file exists download and decompress it
+  if [ $REMOTE_CACHE_FILE_EXISTENCE -eq 0 ];then
+    echo "Downloading cache file: ${REMOTE_CACHE_FILE}..."
+    gsutil -q cp "${REMOTE_CACHE_FILE}" "${SRC_DIR}"
+
+    CACHE_FILE="${SRC_DIR}/${KEY}.tgz"
+
+    echo "Restoring cache from file ${CACHE_FILE}..."
+    tar xpzf "$CACHE_FILE" -P
+    rm "$CACHE_FILE"
+  else
+    # Check if fallback key pattern has been specified
+    if [ -z "${KEY_FALLBACK_PATTERN}" ]; then
+      echo "No fallback key pattern specified. Can not restore cache!"
+      exit 0
+    fi
+
+    # Try to find a fallback file
+    FALLBACK_CACHE_FILE=$(gsutil ls -l $BUCKET/$KEY_FALLBACK_PATTERN** | sort -k 2,2r | grep -m 1 -i "$KEY_FALLBACK_PATTERN" | awk '{print $3}')
+
+    # Check if fallback cache file has been found
+    if [ -z "${FALLBACK_CACHE_FILE}" ]; then
+      echo "No fallback cache file found! Can not restore cache!"
+      exit 0
+    fi
+
+    # Download found fallback cache file
+    echo "Downloading fallback cache file: ${FALLBACK_CACHE_FILE}..."
+    gsutil -q cp "${FALLBACK_CACHE_FILE}" "${SRC_DIR}"
+
+    FALLBACK_CACHE_FILE_NAME=$(echo ${FALLBACK_CACHE_FILE} | awk '{n=split($0,parts,"/"); print parts[n]}')
+
+    echo "Restoring cache from fallback file ${FALLBACK_CACHE_FILE}..."
+    tar xpzf "$FALLBACK_CACHE_FILE_NAME" -P
+    rm "$FALLBACK_CACHE_FILE_NAME"
+  fi
+# If SRC_DIR is specified, restore cache from local file
+elif [ -n "${SRC_DIR}" ]; then
+  LOCAL_CACHE_FILE="${SRC_DIR}/${KEY}.tgz"
+  echo "Restoring from local cache file ${LOCAL_CACHE_FILE}..."
+  tar xpzf "$LOCAL_CACHE_FILE" -P
+fi
diff --git a/cache/save_cache b/cache/save_cache
new file mode 100755
index 0000000000..2c5381786a
--- /dev/null
+++ b/cache/save_cache
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+BUCKET=""
+OUT_DIR="."
+KEY="cache"
+THRESHOLD="50M"
+PATHS=()
+
+function print_usage {
+  echo "Usage: $0 --out=/cache_dir --key=cache-key --path=/path/to/files/1 --path=/path/to/files/2"
+  echo
+  echo "Saves the specified paths to a cache file located in the out directory."
+  echo
+  echo "  -b, --bucket      The cloud storage bucket to upload the cache to. [optional]"
+  echo "  -o, --out         The output directory to write the cache to. [optional]"
+  echo "  -k, --key         The cache key used for this cache file."
+  echo "  -p, --path        The files to store in the cache. Can be repeated."
+  echo "  -t, --threshold   The parallel composite upload threshold [default: 50M]"
+  echo "  -n, --no-clobber  Skips the save if the cache file already exists in GCS."
+  echo
+}
+
+for i in "$@"; do
+  case $i in
+    -b=*|--bucket=*)
+      BUCKET="${i#*=}"
+      shift
+      ;;
+    -o=*|--out=*)
+      OUT_DIR="${i#*=}"
+      shift
+      ;;
+    -k=*|--key=*)
+      KEY="${i#*=}"
+      shift
+      ;;
+    -p=*|--path=*)
+      PATHS+="${i#*=} "
+      shift
+      ;;
+    -t=*|--threshold=*)
+      THRESHOLD="${i#*=} "
+      shift
+      ;;
+    -n|--no-clobber)
+      NO_CLOBBER=true
+      shift
+      ;;
+    -h|--help )
+      print_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: ${i}"
+      print_usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$KEY" ] || [ -z "$PATHS" ];then
+  print_usage
+  exit 1
+fi
+
+if [ "$NO_CLOBBER" = true ] && [ -z "$BUCKET" ];then
+  echo "--bucket must be specified if --no-clobber is used"
+  echo
+  print_usage
+  exit 1
+fi
+
+eval "KEY=$KEY"
+
+CACHE_FILE="${OUT_DIR}/${KEY}.tgz"
+
+if [ "$NO_CLOBBER" = true ];then
+  BUCKET_FILE="$BUCKET/$KEY.tgz"
+  FILE_LS=$(gsutil ls "$BUCKET_FILE")
+  if [ "$FILE_LS" == "$BUCKET_FILE" ];then
+    echo "Cache file exists, exiting save_cache without over-writing cache file."
+    exit 0
+  fi
+fi
+
+echo "Compressing cache to ${CACHE_FILE}..."
+tar cpzf "$CACHE_FILE" ${PATHS[@]} -P
+
+if [ -n "$BUCKET" ];then
+  echo "Uploading cache to Google Cloud Storage..."
+  gsutil -o GSUtil:parallel_composite_upload_threshold=${THRESHOLD} cp -R "$CACHE_FILE" "$BUCKET"
+fi

From bf82cbff81594567f0dcf94fcf699d12e89210f1 Mon Sep 17 00:00:00 2001
From: Kalaiselvi Murugesan <kalaiselvim@google.com>
Date: Wed, 29 Nov 2023 04:25:48 +0000
Subject: [PATCH 2/5] Review changes

---
 build/build-image/cache/Dockerfile-base       |  27 ++++
 build/build-image/cache/Dockerfile-restore    |  18 +++
 build/build-image/cache/Dockerfile-save       |  18 +++
 build/build-image/cache/README.md             | 142 ++++++++++++++++++
 build/build-image/cache/checksum              |  17 +++
 .../build-image/cache}/cloudbuild.yaml        |  15 ++
 .../build-image/cache}/restore_cache          |  14 ++
 {cache => build/build-image/cache}/save_cache |  14 ++
 cache/Dockerfile-base                         |  13 --
 cache/Dockerfile-restore                      |   4 -
 cache/Dockerfile-save                         |   4 -
 cache/checksum                                |   3 -
 12 files changed, 265 insertions(+), 24 deletions(-)
 create mode 100644 build/build-image/cache/Dockerfile-base
 create mode 100644 build/build-image/cache/Dockerfile-restore
 create mode 100644 build/build-image/cache/Dockerfile-save
 create mode 100644 build/build-image/cache/README.md
 create mode 100755 build/build-image/cache/checksum
 rename {cache => build/build-image/cache}/cloudbuild.yaml (85%)
 rename {cache => build/build-image/cache}/restore_cache (84%)
 rename {cache => build/build-image/cache}/save_cache (78%)
 delete mode 100644 cache/Dockerfile-base
 delete mode 100644 cache/Dockerfile-restore
 delete mode 100644 cache/Dockerfile-save
 delete mode 100755 cache/checksum

diff --git a/build/build-image/cache/Dockerfile-base b/build/build-image/cache/Dockerfile-base
new file mode 100644
index 0000000000..c352db1e55
--- /dev/null
+++ b/build/build-image/cache/Dockerfile-base
@@ -0,0 +1,27 @@
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/cloud-builders/gcloud-slim
+
+RUN    apt-get update \
+    && apt-get install -y gcc python3-dev python3-setuptools python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install --no-cache-dir -U crcmod
+
+COPY checksum /bin
+COPY save_cache /bin
+COPY restore_cache /bin
+
+RUN chmod +x /bin/checksum /bin/save_cache /bin/restore_cache
\ No newline at end of file
diff --git a/build/build-image/cache/Dockerfile-restore b/build/build-image/cache/Dockerfile-restore
new file mode 100644
index 0000000000..bd4d33a068
--- /dev/null
+++ b/build/build-image/cache/Dockerfile-restore
@@ -0,0 +1,18 @@
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG project_id
+FROM us-docker.pkg.dev/${project_id}/ci/cache
+
+ENTRYPOINT ["restore_cache"]
\ No newline at end of file
diff --git a/build/build-image/cache/Dockerfile-save b/build/build-image/cache/Dockerfile-save
new file mode 100644
index 0000000000..1941874a13
--- /dev/null
+++ b/build/build-image/cache/Dockerfile-save
@@ -0,0 +1,18 @@
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG project_id
+FROM us-docker.pkg.dev/${project_id}/ci/cache
+
+ENTRYPOINT ["save_cache"]
\ No newline at end of file
diff --git a/build/build-image/cache/README.md b/build/build-image/cache/README.md
new file mode 100644
index 0000000000..d2590a76f8
--- /dev/null
+++ b/build/build-image/cache/README.md
@@ -0,0 +1,142 @@
+# Origin
+
+This `README.md` was originally based on [cloud-builders-community](https://github.com/GoogleCloudPlatform/cloud-builders-community), which contains source code for community-contributed Docker images. You can use these images as build steps for [Google Cloud Build](https://cloud.google.com/build/docs). 
+
+The key change made upon importing the [cache builder's](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache) code into Agones was the update of image tags to use Google Artifact Registry instead of Google Container Registry.
+
+# Cache builders
+
+This includes a pair of builders, `save_cache` and `restore_cache`, that work together to cache files between builds to a GCS bucket (or local file).
+
+## Using the `save_cache` builder
+
+All options that require a value use the form `--option=value` or `-o=value` so that they look nice in Yaml files.
+
+| Option           | Description                                                 |
+| ---------------- | ----------------------------------------------------------- |
+| -b, --bucket     | The cloud storage bucket to upload the cache to. [optional] |
+| -o, --out        | The output directory to write the cache to. [optional]      |
+| -k, --key        | The cache key used for this cache file. [optional]          |
+| -p, --path       | The files to store in the cache. Can be repeated.           |
+| -t, --threshold  | The parallel composite upload threshold [default: 50M]      |
+| -n, --no-clobber | Skips the save if the cache file already exists in GCS.     |
+
+One of `--bucket` or `--out` parameters are required.  If `--bucket` then the cache file will be uploaded to the provided GCS bucket path.  If `--out` then the cache file will be stored in the directory specified on disk.
+
+The key provided by `--key` is used to identify the cache file. Any other cache files for the same key will be overwritten by this one.
+
+The `--path` parameters can be repeated for as many folders as you'd like to cache.  When restored, they will retain folder structure on disk.
+
+The `--no-clobber` flag is used to skip creating and uploading the cache to GCS if the cache file already exists. This will shorten the time for builds when a cache was restored and is not changed by your build process. For example, this flag can be used if you are caching your dependencies and all of your dependencies are pinned to a specific version. This flag is valid only when `--bucket` is used.
+
+## Using the `restore_cache` builder
+
+All options use the form `--option=value` or `-o=value` so that they look nice in Yaml files.
+
+| Option                 | Description                                                                            |
+| ---------------------- | -------------------------------------------------------------------------------------- |
+| -b,  --bucket          | The cloud storage bucket to download the cache from. [optional]                        |
+| -s,  --src             | The local directory in which the cache is stored. [optional]                           |
+| -k,  --key             | The cache key used for this cache file. [optional]                                     |
+| -kf, --key_fallback    | The cache key fallback pattern to be used if exact cache key is not found. [optional]  |
+
+One of `--bucket` or `--src` parameters are required.  If `--bucket` then the cache file will be downloaded from the provided GCS bucket path.  If `--src` then the cache file will be read from the directory specified on disk.
+
+The key provided by `--key` is used to identify the cache file.
+
+The fallback key pattern provide by `--key_fallback`, will be used to fetch the most recent cache file matching that pattern in case there is a cache miss from the specified `--key`.
+
+### `checksum` Helper
+
+As apps develop, cache needs change. For instance when dependencies are removed from a project, or versions are updated, there is no need to cache the older versions of dependencies. Therefore it's recommended that you update the cache key when these changes occur.
+
+This builder includes a `checksum` helper script, which you can use to create a simple checksum of files in your project to use as a cache key.
+
+To use it in the `--key` arguemnt, simply surround the command with `$()`:
+
+```bash
+--key=build-cache-$(checksum build.gradle)-$(checksum dependencies.gradle)
+```
+
+To ensure you aren't paying for storage of obsolete cache files you can add an Object Lifecycle Rule to the cache bucket to delete object older than 30 days.
+
+## Examples
+
+The following examples demonstrate build requests that use this builder.
+
+### Saving a cache with checksum to GCS bucket
+
+This `cloudbuild.yaml` saves the files and folders in the `path` arguments to a cache file in the GCS bucket `gs://$CACHE_BUCKET/`.  In this example the key will be updated, resulting in a new cache, every time the `cloudbuild.yaml` build file is changed.
+
+```yaml
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  args:
+  - '--bucket=gs://$CACHE_BUCKET/'
+  - '--key=resources-$( checksum cloudbuild.yaml )'
+  - '--path=.cache/folder1'
+  - '--path=.cache/folder2/subfolder3'
+```
+
+If your build process only changes the cache contents whenever `cloudbuild.yaml` changes, then you can skip saving the cache again if it already exists in GCS:
+```yaml
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  args:
+  - '--bucket=gs://$CACHE_BUCKET/'
+  - '--key=resources-$( checksum cloudbuild.yaml )'
+  - '--path=.cache/folder1'
+  - '--path=.cache/folder2/subfolder3'
+  - '--no-clobber'
+```
+
+### Saving a cache with checksum to a local file
+
+This `cloudbuild.yaml` saves the files and folders in the `path` arguments to a cache file in the directory passed to the `out` parameter.  In this example the key will be updated, resulting in a new cache, every time the `cloudbuild.yaml` build file is changed.
+
+```yaml
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  args:
+  - '--out=/cache/'
+  - '--key=resources-$( checksum cloudbuild.yaml )'
+  - '--path=.cache/folder1'
+  - '--path=.cache/folder2/subfolder3'
+  volumes:
+  - name: 'cache'
+    path: '/cache'
+```
+
+### Restore a cache from a GCS bucket
+
+This `cloudbuild.yaml` restores the files from the compressed cache file identified by `key` on the cache bucket provided, if it exists.
+
+```yaml
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+  args:
+  - '--bucket=gs://$CACHE_BUCKET/'
+  - '--key=resources-$( checksum cloudbuild.yaml )'
+```
+
+### Restore a cache from a local file
+
+This `cloudbuild.yaml` restores the files from the compressed cache file identified by `key` on the local filesystem, if it exists.
+
+```yaml
+- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+  args:
+  - '--src=/cache/'
+  - '--key=resources-$( checksum cloudbuild.yaml )'
+  volumes:
+  - name: 'cache'
+    path: '/cache'
+```
+
+### Restore a cache with a fallback key
+
+```yaml
+- name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache
+  id: restore_cache
+  args: [
+    '--bucket=gs://${_CACHE_BUCKET}',
+    '--key=gradle-$( checksum checksum.txt )',
+    '--key_fallback=gradle-',
+  ]
+```
diff --git a/build/build-image/cache/checksum b/build/build-image/cache/checksum
new file mode 100755
index 0000000000..5dbe2e9438
--- /dev/null
+++ b/build/build-image/cache/checksum
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo $(cksum $@ | cut -d' ' -f1) | tr ' ' -
diff --git a/cache/cloudbuild.yaml b/build/build-image/cache/cloudbuild.yaml
similarity index 85%
rename from cache/cloudbuild.yaml
rename to build/build-image/cache/cloudbuild.yaml
index b6d235ba44..3d91bba036 100644
--- a/cache/cloudbuild.yaml
+++ b/build/build-image/cache/cloudbuild.yaml
@@ -1,3 +1,18 @@
+---
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 steps:
 
 - name: make-docker
diff --git a/cache/restore_cache b/build/build-image/cache/restore_cache
similarity index 84%
rename from cache/restore_cache
rename to build/build-image/cache/restore_cache
index 94dfb5a027..69b339f69d 100755
--- a/cache/restore_cache
+++ b/build/build-image/cache/restore_cache
@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
 
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 BUCKET=""
 SRC_DIR="."
 KEY="cache"
diff --git a/cache/save_cache b/build/build-image/cache/save_cache
similarity index 78%
rename from cache/save_cache
rename to build/build-image/cache/save_cache
index 2c5381786a..8b8b217e3e 100755
--- a/cache/save_cache
+++ b/build/build-image/cache/save_cache
@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
 
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 BUCKET=""
 OUT_DIR="."
 KEY="cache"
diff --git a/cache/Dockerfile-base b/cache/Dockerfile-base
deleted file mode 100644
index 06228044d5..0000000000
--- a/cache/Dockerfile-base
+++ /dev/null
@@ -1,13 +0,0 @@
-FROM gcr.io/cloud-builders/gcloud-slim
-
-RUN    apt-get update \
-    && apt-get install -y gcc python3-dev python3-setuptools python3-pip \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip3 install --no-cache-dir -U crcmod
-
-COPY checksum /bin
-COPY save_cache /bin
-COPY restore_cache /bin
-
-RUN chmod +x /bin/checksum /bin/save_cache /bin/restore_cache
\ No newline at end of file
diff --git a/cache/Dockerfile-restore b/cache/Dockerfile-restore
deleted file mode 100644
index 7ffc911d53..0000000000
--- a/cache/Dockerfile-restore
+++ /dev/null
@@ -1,4 +0,0 @@
-ARG project_id
-FROM us-docker.pkg.dev/${project_id}/ci/cache
-
-ENTRYPOINT ["restore_cache"]
\ No newline at end of file
diff --git a/cache/Dockerfile-save b/cache/Dockerfile-save
deleted file mode 100644
index 4d62fa36ee..0000000000
--- a/cache/Dockerfile-save
+++ /dev/null
@@ -1,4 +0,0 @@
-ARG project_id
-FROM us-docker.pkg.dev/${project_id}/ci/cache
-
-ENTRYPOINT ["save_cache"]
\ No newline at end of file
diff --git a/cache/checksum b/cache/checksum
deleted file mode 100755
index 6ac8149d5d..0000000000
--- a/cache/checksum
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-echo $(cksum $@ | cut -d' ' -f1) | tr ' ' -

From 96d748e0e05d92d7081da91ca618a1e03f008539 Mon Sep 17 00:00:00 2001
From: Kalaiselvi Murugesan <kalaiselvim@google.com>
Date: Wed, 29 Nov 2023 04:53:54 +0000
Subject: [PATCH 3/5] fix lint

---
 build/build-image/cache/cloudbuild.yaml | 242 ++++++++++++------------
 1 file changed, 120 insertions(+), 122 deletions(-)

diff --git a/build/build-image/cache/cloudbuild.yaml b/build/build-image/cache/cloudbuild.yaml
index 3d91bba036..dcb6a1384e 100644
--- a/build/build-image/cache/cloudbuild.yaml
+++ b/build/build-image/cache/cloudbuild.yaml
@@ -15,134 +15,132 @@
 
 steps:
 
-- name: make-docker
-  id: build_base_image
-  args: 
-  - 'build'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
-  - '--file=Dockerfile-base'
-  - '.'
+  - name: 'gcr.io/cloud-builders/docker'
+    id: build_base_image
+    args:
+      - 'build'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+      - '--file=Dockerfile-base'
+      - '.'
 
-- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
-  id: build_save_cache
-  args: 
-  - 'build'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
-  - '--file=Dockerfile-save'
-  - '--build-arg=project_id=$PROJECT_ID'
-  - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
-  - '.'
+  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+    id: build_save_cache
+    args:
+      - 'build'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
+      - '--file=Dockerfile-save'
+      - '--build-arg=project_id=$PROJECT_ID'
+      - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+      - '.'
 
-- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
-  id: build_restore_cache
-  args: 
-  - 'build'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
-  - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'
-  - '--file=Dockerfile-restore'
-  - '--build-arg=project_id=$PROJECT_ID'
-  - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
-  - '.'
+  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+    id: build_restore_cache
+    args:
+      - 'build'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
+      - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'
+      - '--file=Dockerfile-restore'
+      - '--build-arg=project_id=$PROJECT_ID'
+      - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+      - '.'
 
-# Test the script
-- name: 'gcr.io/cloud-builders/gsutil'
-  id: setup_test
-  entrypoint: 'bash'
-  args: 
-  - '-c'
-  - |
-    echo "Creating test cache file structure."
-    mkdir -p /original/folder1 /original/folder2/subfolder3 rel_folder
-    touch /original/folder1/file1.txt
-    touch /original/folder1/file2.txt
-    touch /original/folder2/ignore.txt
-    touch /original/folder2/subfolder3/file1.txt
-    touch rel_folder/file3.txt
-  volumes:
-  - name: original
-    path: /original
+  # Test the script
+  - name: 'gcr.io/cloud-builders/gsutil'
+    id: setup_test
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        echo "Creating test cache file structure."
+        mkdir -p /original/folder1 /original/folder2/subfolder3 rel_folder
+        touch /original/folder1/file1.txt
+        touch /original/folder1/file2.txt
+        touch /original/folder2/ignore.txt
+        touch /original/folder2/subfolder3/file1.txt
+        touch rel_folder/file3.txt
+    volumes:
+      - name: original
+        path: /original
 
-- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
-  id: save_cache
-  args:
-  - '--out=/cached'
-  - '--key=simple-key-$( checksum cloudbuild.yaml )'
-  - '--path=/original/folder1'
-  - '--path=/original/folder2/subfolder3'
-  - '--path=rel_folder'
-  volumes:
-  - name: original
-    path: /original
-  - name: cached
-    path: /cached
+  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+    id: save_cache
+    args:
+      - '--out=/cached'
+      - '--key=simple-key-$( checksum cloudbuild.yaml )'
+      - '--path=/original/folder1'
+      - '--path=/original/folder2/subfolder3'
+      - '--path=rel_folder'
+    volumes:
+      - name: original
+        path: /original
+      - name: cached
+        path: /cached
 
-- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
-  id: verify_cache
-  entrypoint: 'bash'
-  args:
-  - '-c'
-  - |
-    echo "Verifying cache file exists."
+  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+    id: verify_cache
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        echo "Verifying cache file exists."
+        cache_file="/cached/simple-key-$( checksum cloudbuild.yaml ).tgz"
+        if [[ ! -f "${cache_file}" ]];then
+          echo "Missing cache file at ${cache_file}"
+          echo "Contents:"
+          echo "$(ls -al /cached)"
+          exit 1
+        fi
+        echo "Cache tests passed."
+    volumes:
+      - name: cached
+        path: /cached
 
-    cache_file="/cached/simple-key-$( checksum cloudbuild.yaml ).tgz"
-    if [[ ! -f "${cache_file}" ]];then
-      echo "Missing cache file at ${cache_file}"
-      echo "Contents:"
-      echo "$(ls -al /cached)"
-      exit 1
-    fi
+  - name: 'gcr.io/cloud-builders/gsutil'
+    id: clean_cache
+    entrypoint: bash
+    args:
+      - -c
+      - |
+        echo "Clearing original files..."
+        rm -rf /original/*
+        rm -rf rel_folder/
+    volumes:
+      - name: original
+        path: /original
 
-    echo "Cache tests passed."
-  volumes:
-  - name: cached
-    path: /cached
+  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+    id: restore_cache
+    args:
+      - '--src=/cached'
+      - '--key=simple-key-$( checksum cloudbuild.yaml )'
+    volumes:
+      - name: original
+        path: /original
+      - name: cached
+        path: /cached
 
-- name: 'gcr.io/cloud-builders/gsutil'
-  id: clean_cache
-  entrypoint: bash
-  args:
-  - -c
-  - |
-    echo "Clearing original files..."
-    rm -rf /original/*
-    rm -rf rel_folder/
-  volumes:
-  - name: original
-    path: /original
+  - name: 'gcr.io/cloud-builders/gsutil'
+    id: verify_restore
+    entrypoint: bash
+    args:
+      - '-c'
+      - |
+        test -f /original/folder1/file1.txt
+        test -f /original/folder1/file2.txt
+        test -f /original/folder2/ignore.txt
+        test -f /original/folder2/subfolder3/file1.txt
+        test -f rel_folder/file3.txt
+        test -f /workspace/rel_folder/file3.txt
+    volumes:
+      - name: original
+        path: /original
 
-- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
-  id: restore_cache
-  args:
-  - '--src=/cached'
-  - '--key=simple-key-$( checksum cloudbuild.yaml )'
-  volumes:
-  - name: original
-    path: /original
-  - name: cached
-    path: /cached
-
-- name: 'gcr.io/cloud-builders/gsutil'
-  id: verify_restore
-  entrypoint: bash
-  args:
-  - '-c'
-  - |
-    test -f /original/folder1/file1.txt
-    test -f /original/folder1/file2.txt
-    test -f /original/folder2/ignore.txt
-    test -f /original/folder2/subfolder3/file1.txt
-    test -f rel_folder/file3.txt
-    test -f /workspace/rel_folder/file3.txt
-  volumes:
-  - name: original
-    path: /original
-
-images: 
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
-- 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'
+images:
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}'
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}'
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest'
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}'
+  - 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest'

From d61a476b3e7162950a93486a5c53d5bee4cf90a0 Mon Sep 17 00:00:00 2001
From: Kalaiselvi Murugesan <kalaiselvim@google.com>
Date: Fri, 1 Dec 2023 01:32:08 +0000
Subject: [PATCH 4/5] Review update

---
 build/build-image/cache/cloudbuild.yaml | 4 ++--
 build/includes/release.mk               | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/build/build-image/cache/cloudbuild.yaml b/build/build-image/cache/cloudbuild.yaml
index dcb6a1384e..d78ccefa86 100644
--- a/build/build-image/cache/cloudbuild.yaml
+++ b/build/build-image/cache/cloudbuild.yaml
@@ -24,7 +24,7 @@ steps:
       - '--file=Dockerfile-base'
       - '.'
 
-  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache'
+  - name: 'gcr.io/cloud-builders/docker'
     id: build_save_cache
     args:
       - 'build'
@@ -35,7 +35,7 @@ steps:
       - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest'
       - '.'
 
-  - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache'
+  - name: 'gcr.io/cloud-builders/docker'
     id: build_restore_cache
     args:
       - 'build'
diff --git a/build/includes/release.mk b/build/includes/release.mk
index 2e3f9b98cc..61fe924745 100644
--- a/build/includes/release.mk
+++ b/build/includes/release.mk
@@ -87,3 +87,8 @@ pre-build-release:
 post-build-release:
 	docker run --rm $(common_mounts) -w $(workdir_path)/build/release $(build_tag) \
 		gcloud builds submit . --substitutions _VERSION=$(base_version) --config=./post_cloudbuild.yaml $(ARGS)
+
+# Ensure that ci/save_cache and ci/restore_cache are not deleted by the cleanup policy
+cache-save-restore:
+	docker run --rm $(common_mounts) -w $(workdir_path)/build/build-image/cache $(build_tag) \
+		gcloud builds submit . --substitutions _VERSION=$(base_version) --config=./cloudbuild.yaml $(ARGS)

From c3b10337d527881a469eb27333c268cd6e043fc7 Mon Sep 17 00:00:00 2001
From: Kalaiselvi Murugesan <kalaiselvim@google.com>
Date: Fri, 1 Dec 2023 21:57:36 +0000
Subject: [PATCH 5/5] Remove target

---
 build/includes/release.mk | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/build/includes/release.mk b/build/includes/release.mk
index 61fe924745..2e3f9b98cc 100644
--- a/build/includes/release.mk
+++ b/build/includes/release.mk
@@ -87,8 +87,3 @@ pre-build-release:
 post-build-release:
 	docker run --rm $(common_mounts) -w $(workdir_path)/build/release $(build_tag) \
 		gcloud builds submit . --substitutions _VERSION=$(base_version) --config=./post_cloudbuild.yaml $(ARGS)
-
-# Ensure that ci/save_cache and ci/restore_cache are not deleted by the cleanup policy
-cache-save-restore:
-	docker run --rm $(common_mounts) -w $(workdir_path)/build/build-image/cache $(build_tag) \
-		gcloud builds submit . --substitutions _VERSION=$(base_version) --config=./cloudbuild.yaml $(ARGS)