Merge pull request #6 from Sage/cache-to-from

Enable --cache-from and --cache-to

README.md

@@ -1,2 +1,2 @@
 # sbc-shared-buildkite-plugin
-Collection of Sage specific functions used with Buildkite 
+Collection of Sage specific functions used with Buildkite

hooks/command

@@ -3,4 +3,4 @@ set -euo pipefail
 if [[ -n $BUILDKITE_COMMAND ]]; then
   echo "Step command detected for $BUILDKITE_LABEL .  Executing command"
   eval $BUILDKITE_COMMAND
-fi
+fi

hooks/post-command

@@ -30,15 +30,15 @@ if [[ $ACTION == "push_image" ]];then
       fi
     fi
   fi
-elif [[ $ACTION == "push_param" ]];then
+elif [[ $ACTION == "push_param" ]]; then
  config_path=$ENVIRONMENT/$LANDSCAPE
 
   # change directory from which all param logic expects to be run from.
   cd configuration
  ./push.sh $config_path
-elif [[ $ACTION == "publish_gem" ]];then
+elif [[ $ACTION == "publish_gem" ]]; then
   echo ""
-elif [[ $ACTION == "build" ]];then
+elif [[ $ACTION == "build" ]]; then
   # Load a predetermined custom functions file that a repository may need to use to extend the build process
   CUSTOM_FILE=".buildkite/custom_functions.sh"
   if [ -f "$CUSTOM_FILE" ]; then

lib/functions.sh

@@ -11,8 +11,6 @@ setup() {
     exit 1
   fi
 
-  echo "Setting up variables for $1"
-
   # Setup the env that contains the application name and repo name
   export APP=$(cat .buildkite/.application)
   export REPO=$1/$APP
@@ -30,11 +28,14 @@ setup() {
   fi
 
   export BK_ECR=268539851198.dkr.ecr.eu-west-1.amazonaws.com/sageone/buildkite
+  export BK_CACHE=268539851198.dkr.ecr.eu-west-1.amazonaws.com/sageone/cache
+
+  # Needed for --cache-from and --cache-to
+  docker buildx create --use --bootstrap
 }
 
 # convert --<switch> to a variable
 switches() {
-  echo "Processing switches"
   while [ $# -gt 0 ]; do
    if [[ $1 == *"--"* ]]; then
         v="${1/--/}"
@@ -47,8 +48,6 @@ switches() {
 
 # validate list of switch names exist as a set variable
 validate_switches() {
-  echo "Validating switches"
-
   arr=("$@")
 
   set +u
@@ -65,16 +64,17 @@ validate_switches() {
 }
 
 # target => (Optional) set the target build stage to build
-# tag => tag for the docker image
-# file => source docker file to build from
-# cache_id => cache identifier from where it was built from.  Typically GH branch name
+# tag => variant of the docker image e.g. app or database
+# file => source Dockerfile
+# cache_id => typically the git branch name
 buildx() {
   target=
   switches "$@"
   validate_switches tag file cache_id
   varx REPO
+  varx BUILDKITE_PIPELINE_DEFAULT_BRANCH
 
-  echo "--- :building_construction: Build $tag"
+  echo "+++ :building_construction: Build $tag"
 
   local OPTIONAL_TARGET=
   if [[ -n $target ]]; then
@@ -83,38 +83,20 @@ buildx() {
 
   docker buildx build \
     -f $file \
-    --build-arg BUILDKIT_INLINE_CACHE=1 \
     --build-arg CI_BRANCH \
     --build-arg CI_STRING_TIME \
-    --cache-from $BK_ECR:$APP-$tag-cache-$cache_id \
-    --cache-from $BK_ECR:$APP-$tag-cache-master \
+    --cache-to mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=$BK_CACHE:$APP-$tag-$cache_id \
+    --cache-from $BK_CACHE:$APP-$tag-$cache_id \
+    --cache-from $BK_CACHE:$APP-$tag-$BUILDKITE_PIPELINE_DEFAULT_BRANCH \
     --secret id=railslts,env=BUNDLE_GEMS__RAILSLTS__COM \
     --secret id=jfrog,env=BUNDLE_SAGEONEGEMS__JFROG__IO \
-    --ssh default $OPTIONAL_TARGET \
+    --ssh default \
+    $OPTIONAL_TARGET \
+    --load \
     -t $REPO:$tag \
     .
 }
 
-# app => name of the application
-# target => (Optional) set the target build stage to build
-# tag => tag for the docker image
-# file => source docker file to build from
-# cache_id => cache identifier from where it was built from.  Typically GH branch name
-buildx_and_cachex () {
-  target=
-  switches "$@"
-  validate_switches app tag cache_id file
-  varx REPO
-
-  local OPTIONAL_TARGET=
-  if [[ -n $target ]]; then
-    OPTIONAL_TARGET="--target $target"
-  fi
-
-  buildx --app $app $OPTIONAL_TARGET --tag $tag --file $file --cache_id $cache_id
-
-  cachex --app $app --tag $tag --cache_id $cache_id
-}
 
 # Push an image into the BK ECR
 pushx () {
@@ -129,18 +111,6 @@ pushx () {
   docker push $BUILD_IMAGE_NAME
 }
 
-# Push an image into the BK ECR for caching builds
-cachex () {
-  switches "$@"
-  validate_switches app tag cache_id
-  varx REPO
-
-  echo "--- :s3: Cache $tag"
-  local BUILD_IMAGE_NAME=$BK_ECR:$app-$tag-cache-$cache_id
-  docker tag $REPO:$tag $BUILD_IMAGE_NAME
-  docker push $BUILD_IMAGE_NAME
-}
-
 # Push an image into a target ECR for deployments
 push_image () {
   switches "$@"

plugin.yml

@@ -25,4 +25,4 @@ configuration:
       type: boolean
   required:
     - action
-  additionalProperties: false
+  additionalProperties: false