Cache CI runs to reduce run times by ~5 minutes

This work is being copied across from our Rails Template. It was added after this project was created [1].

We are already using Docker Compose in CI on this project. However for context there is another pending change on the Rails Template[2] that includes a screenshot of how caching can behave.

The approach here is that we create a shared area on disk called /tmp/docker-save. We load this in at the start of every test run. It will make a past docker image for this app available to the build context. Docker can then use this image and it's layers to optimise the build steps, opting for the cached version rather than rebuilding from scratch.

If the cache is not hit (meaning the Dockerfile or gem.lock file changed) then it will add a new image into the cache at the end of the run. In this situation the builds will still take as long as they do now however this should be much less frequent.

[1] dxw/rails-template#147
[2] dxw/rails-template#213

.github/workflows/continuous-integration.yml

@@ -13,7 +13,22 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+      - id: cache-docker
+        uses: actions/cache@v2
+        with:
+          path: /tmp/docker-save
+          key:
+            docker-save-${{ hashFiles('Dockerfile', 'Gemfile.lock',
+            'package-lock.json') }}
+      - name: Load cached Docker image
+        run: docker load -i /tmp/docker-save/snapshot.tar || true
+        if: steps.cache-docker.outputs.cache-hit == 'true'
       - name: Build
-        run: docker-compose -f docker-compose.ci.yml build
+        run: docker-compose -f docker-compose.ci.yml -p app build
       - name: Test
         run: docker-compose -f docker-compose.ci.yml run --rm test script/test
+      - name: Prepare Docker cache
+        run:
+          mkdir -p /tmp/docker-save && docker save app_test:latest -o
+          /tmp/docker-save/snapshot.tar && ls -lh /tmp/docker-save
+        if: always() && steps.cache-docker.outputs.cache-hit != 'true'

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog 1.0.0].
 - existing specification page displays useful message when no specs exist
 - fix text input field width to fit full screen width
 - document where to find the service in the readme
+- cache CI builds to reduce build times
 
 ## [release-009] - 2021-05-21
 

Dockerfile

@@ -1,69 +1,89 @@
-# BUILD STAGE #
-FROM ruby:2.6.6 AS build
+# ------------------------------------------------------------------------------
+# Base
+# ------------------------------------------------------------------------------
+FROM ruby:2.6.6 as base
 MAINTAINER dxw <rails@dxw.com>
 
-ENV INSTALL_PATH /srv/app
-ARG RAILS_ENV
-ENV RAILS_ENV=${RAILS_ENV:-production}
-ENV RACK_ENV=${RAILS_ENV:-production}
-
-WORKDIR $INSTALL_PATH
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash
+RUN curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 
 RUN apt-get update && apt-get install -qq -y \
   build-essential \
   libpq-dev \
   --fix-missing --no-install-recommends
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
-        && apt-get install -y nodejs
 
-COPY package.json ./package.json
-COPY package-lock.json ./package-lock.json
+ENV APP_HOME /srv/app
+ENV DEPS_HOME /deps
+
+ARG RAILS_ENV
+ENV RAILS_ENV ${RAILS_ENV:-production}
+ENV NODE_ENV ${RAILS_ENV:-production}
+
+# ------------------------------------------------------------------------------
+# Dependencies
+# ------------------------------------------------------------------------------
+FROM base AS dependencies
+
+RUN mkdir -p ${DEPS_HOME}
+WORKDIR $DEPS_HOME
 
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+  && apt-get install -y nodejs \
+  && npm install --global yarn
+
+# Install Javascript dependencies
+COPY package-lock.json $DEPS_HOME/package-lock.json
+COPY package.json $DEPS_HOME/package.json
 RUN npm install
 
-COPY Gemfile* ./
-RUN gem install bundler:2.1.4 --no-document
+# Install Ruby dependencies
+COPY Gemfile $DEPS_HOME/Gemfile
+COPY Gemfile.lock $DEPS_HOME/Gemfile.lock
+RUN gem update --system
+RUN gem install bundler -v 2.2.16
 
-ARG BUNDLE_EXTRA_GEM_GROUPS
-ENV BUNDLE_GEM_GROUPS=${BUNDLE_EXTRA_GEM_GROUPS:-"production"}
+ENV BUNDLE_GEM_GROUPS=$RAILS_ENV
+RUN bundle config set frozen "true"
 RUN bundle config set no-cache "true"
 RUN bundle config set with $BUNDLE_GEM_GROUPS
-RUN bundle install --no-binstubs --retry=3 --jobs=4
+RUN bundle install --no-binstubs --retry=10 --jobs=4
 
-# Copy app code (sorted by vague frequency of change for caching)
-RUN mkdir -p ${INSTALL_PATH}/log
-RUN mkdir -p ${INSTALL_PATH}/tmp
-
-COPY config.ru ${INSTALL_PATH}/config.ru
-COPY Rakefile ${INSTALL_PATH}/Rakefile
-
-COPY public ${INSTALL_PATH}/public
-COPY vendor ${INSTALL_PATH}/vendor
-COPY bin ${INSTALL_PATH}/bin
-COPY lib ${INSTALL_PATH}/lib
-COPY config ${INSTALL_PATH}/config
-COPY db ${INSTALL_PATH}/db
-COPY script ${INSTALL_PATH}/script
-COPY spec ${INSTALL_PATH}/spec
-COPY app ${INSTALL_PATH}/app
-# End
-
-# RELEASE STAGE #
-FROM ruby:2.6.6 AS release
+# ------------------------------------------------------------------------------
+# Web
+# ------------------------------------------------------------------------------
+FROM dependencies AS web
 
-ENV INSTALL_PATH /srv/app
-ARG RAILS_ENV
-ENV RAILS_ENV=${RAILS_ENV:-production}
-ENV RACK_ENV=${RAILS_ENV:-production}
+RUN mkdir -p ${APP_HOME}
+WORKDIR ${APP_HOME}
 
-WORKDIR $INSTALL_PATH
+# Copy app code (sorted by vague frequency of change for caching)
+RUN mkdir -p ${APP_HOME}/log
+RUN mkdir -p ${APP_HOME}/tmp
+
+COPY config.ru ${APP_HOME}/config.ru
+COPY Rakefile ${APP_HOME}/Rakefile
+
+COPY Gemfile $APP_HOME/Gemfile
+COPY Gemfile.lock $APP_HOME/Gemfile.lock
+
+COPY public ${APP_HOME}/public
+COPY vendor ${APP_HOME}/vendor
+COPY bin ${APP_HOME}/bin
+COPY lib ${APP_HOME}/lib
+COPY config ${APP_HOME}/config
+COPY db ${APP_HOME}/db
+COPY script ${APP_HOME}/script
+COPY app ${APP_HOME}/app
+# End
 
-RUN gem install bundler:2.1.4 --no-document
+# Create tmp/pids
+RUN mkdir -p tmp/pids
 
-COPY --from=build /usr/local/bundle/ /usr/local/bundle/
-COPY --from=build $INSTALL_PATH $INSTALL_PATH
+# This must be ordered before rake assets:precompile
+RUN cp -R $DEPS_HOME/node_modules $APP_HOME/node_modules
+RUN cp -R $DEPS_HOME/node_modules/govuk-frontend/govuk/assets $APP_HOME/app/assets
 
-# Compiling assets requires a key to exist: https://github.com/rails/rails/issues/32947
 RUN if [ "$RAILS_ENV" = "production" ]; then \
       RAILS_ENV=production SECRET_KEY_BASE="key" bundle exec rake assets:precompile; \
     fi
@@ -75,3 +95,16 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 EXPOSE 3000
 
 CMD ["bundle", "exec", "rails", "server"]
+
+# ------------------------------------------------------------------------------
+# Test
+# ------------------------------------------------------------------------------
+FROM web as test
+
+RUN apt-get install -qq -y shellcheck
+
+COPY package.json ${APP_HOME}/package.json
+COPY package-lock.json ${APP_HOME}/package-lock.json
+
+COPY .rspec ${APP_HOME}/.rspec
+COPY spec ${APP_HOME}/spec

docker-compose.ci.yml

@@ -3,9 +3,12 @@ services:
   test:
     build:
       context: .
+      target: test
       args:
         RAILS_ENV: "test"
-        BUNDLE_EXTRA_GEM_GROUPS: "test"
+      cache_from:
+        - app_test:latest
+    image: app_test
     command: bundle exec rake
     ports:
       - "3000:3000"

docker-compose.test.yml

@@ -3,9 +3,9 @@ services:
   test:
     build:
       context: .
+      target: test
       args:
         RAILS_ENV: "test"
-        BUNDLE_EXTRA_GEM_GROUPS: "test"
     command: bundle exec rake
     ports:
       - "3000:3000"

docker-compose.yml

@@ -3,9 +3,9 @@ services:
   web:
     build:
       context: .
+      target: web
       args:
         RAILS_ENV: "development"
-        BUNDLE_EXTRA_GEM_GROUPS: "development test"
     command: bash -c "rm -f tmp/pids/server.pid && bundle exec rails s -p 3000 -b '0.0.0.0'"
     ports:
       - "3000:3000"