Set up CircleCI (including [github] tests) (#1338)

I don’t like that our build goes red on master all the time due to flaky service tests. I thought I’d look into other CI services that would make it possible to run the scheduled tests nightly without causing those messages to show up.

CircleCI, Heroku CI, and Codeship were obvious choices. Heroku CI wasn’t free and I didn’t have any experience with Codeship, so I looked into CircleCI. I’ve used their 1.0 system a lot though this was my first time on their 2.0 system. As with earlier versions, they’ve put a lot of work into making the build fast – perhaps more than any other CI system I’ve seen.

I had such good results, my goal shifted from scheduled daily builds (that don’t litter our commit history with red builds) to improving the CI experience as a whole.

This change made a big impact:

- Build logs load much, much faster. In the test I just ran, 22 seconds to < 2 seconds, a 90% improvement.
- Status of each step shows up right in the GitHub UI, which makes it much faster to see exactly what’s failed.
- Builds run about 50-75% faster on account of parallelism.
- GitHub service tests are fixed. This has been a long-standing issue.
- Ability to ssh into a build container to debug failures.

Here’s what I did:

- Created custom Docker images with our dependencies. To be honest, I’m not even sure these are necessary, only to install the greenkeeper-lockfile. We could get dejavu from npm. They make startup very fast.
- Created an npm-install stage which loads all dependencies into node_modules and caches them.
- Created separate stages for our main tests, service tests, and frontend tests, and stages to run the main tests and service tests in Node 6. These run in parallel, up to four at a time.
- Separated service test ID output from the service test results themselves. (I check these often during the PR process, when I confirm that service tests actually ran. Because the production Shields server caches the title, after updating it you can’t tell whether the update is taking effect.)
- Added a personal access token for the shields-ci user. This should actually fix the long-standing issue #979. CircleCI provides an option to “Pass secrets to builds from forked pull requests,” which means unlike Travis, they’ll give us enough rope to shoot ourselves in the foot.
- Schedule a daily build, which runs all the service tests.

.circleci/config.yml

@@ -0,0 +1,214 @@
+version: 2
+
+jobs:
+  npm-install:
+    docker:
+      - image: shieldsio/shields-node-8:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "package.json" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: Install dependencies
+          command: npm install
+
+      - save_cache:
+          paths:
+            - node_modules
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Update Greenkeeper lockfile
+          command: |
+            greenkeeper-lockfile-update
+
+  main:
+    docker:
+      - image: shieldsio/shields-node-8:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Linter
+          when: always
+          command: npm run lint
+
+      - run:
+          name: Server unit tests
+          when: always
+          command: npm run test:js:server
+
+      - run:
+          name: Upload Greenkeeper lockfile
+          command: greenkeeper-lockfile-upload
+
+  main@node-6:
+    docker:
+      - image: shieldsio/shields-node-6:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Linter
+          when: always
+          command: npm run lint
+
+      - run:
+          name: Server unit tests
+          when: always
+          command: npm run test:js:server
+
+      - run:
+          name: Upload Greenkeeper lockfile
+          command: greenkeeper-lockfile-upload
+
+  frontend:
+    docker:
+      - image: shieldsio/shields-node-8:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Frontend unit tests
+          when: always
+          command: npm run test:js:frontend
+
+      - run:
+          name: Frontend build completes successfully
+          when: always
+          command: npm run build
+
+  services-pr:
+    docker:
+      - image: shieldsio/shields-node-8:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - run:
+          name: Prepare service tests
+          command: |
+            mkdir private
+            echo "{\"gh_token\":\"$GITHUB_TOKEN\"}" > private/secret.json
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Identify services tagged in the PR title
+          command: |
+            if [[ ! -z $CI_PULL_REQUEST ]]; then
+              npm run test:services:pr:prepare
+            else
+              echo 'This is not a pull request. Skipping.'
+            fi
+
+      - run:
+          name: Run tests for tagged services
+          command: |
+            if [[ ! -z $CI_PULL_REQUEST ]]; then
+              npm run test:services:pr:run
+            else
+              echo 'This is not a pull request. Skipping.'
+            fi
+
+  services-pr@node-6:
+    docker:
+      - image: shieldsio/shields-node-6:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - run:
+          name: Prepare service tests
+          command: |
+            mkdir private
+            echo "{\"gh_token\":\"$GITHUB_TOKEN\"}" > private/secret.json
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Identify services tagged in the PR title
+          command: |
+            if [[ ! -z $CI_PULL_REQUEST ]]; then
+              npm run test:services:pr:prepare
+            else
+              echo 'This is not a pull request. Skipping.'
+            fi
+
+      - run:
+          name: Run tests for tagged services
+          command: |
+            if [[ ! -z $CI_PULL_REQUEST ]]; then
+              npm run test:services:pr:run
+            else
+              echo 'This is not a pull request. Skipping.'
+            fi
+
+  services-daily:
+    docker:
+      - image: shieldsio/shields-node-8:0.0.1
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          key: v1-dependencies-{{ checksum "package.json" }}
+
+      - run:
+          name: Run all service tests
+          command: npm run test:services
+
+workflows:
+  version: 2
+
+  on-commit:
+    jobs:
+      - npm-install:
+          filters:
+            branches:
+              ignore: gh-pages
+      - main:
+          requires:
+            - npm-install
+      - main@node-6:
+          requires:
+            - npm-install
+      - frontend:
+          requires:
+            - npm-install
+      - services-pr:
+          requires:
+            - npm-install
+      - services-pr@node-6:
+          requires:
+            - npm-install
+
+  daily:
+    triggers:
+      - schedule:
+          cron: "0 17 * * *"
+          filters:
+            branches:
+              only: master
+    jobs:
+      - services-daily

.circleci/images/README.md

@@ -0,0 +1,32 @@
+Updating CircleCI Docker images
+===============================
+
+Prerequisites
+-------------
+
+1. Ask @paulmelnikow to be added to the shieldsio organization on DockerHub.
+2. Install Docker. I tested [these instructions on OS X][Install Docker on OS X].
+3. Run `eval $(docker-machine env default)`
+   (In fish: `eval (docker-machine env default)`)
+
+[Install Docker on OS X]: https://pilsniak.com/how-to-install-docker-on-mac-os-using-brew/
+
+Updating the images
+-------------------
+
+Note: Increment the patch version on the tag in each change.
+
+```console
+IMAGE_TAG=<version> npm run circle-images:build
+docker login
+IMAGE_TAG=<version> npm run circle-images:push
+```
+
+After pushing the images, bump the tag in `.circleci/config.yml`.
+
+Reference
+---------
+
+For more details see the [CircleCI custom image docs][].
+
+[CircleCI custom image docs]: https://circleci.com/docs/2.0/custom-images/

.circleci/images/node-6/Dockerfile

@@ -0,0 +1,4 @@
+FROM node:6
+ADD .circleci/images/prepare-container.sh /root/prepare-container.sh
+RUN /root/prepare-container.sh
+RUN rm /root/prepare-container.sh

.circleci/images/node-8/Dockerfile

@@ -0,0 +1,4 @@
+FROM node:8
+ADD .circleci/images/prepare-container.sh /root/prepare-container.sh
+RUN /root/prepare-container.sh
+RUN rm /root/prepare-container.sh

.circleci/images/prepare-container.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eo pipefail
+
+apt-get -y update
+apt-get install -y --no-install-recommends fonts-dejavu-core
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+npm install -g greenkeeper-lockfile@1

README.md

@@ -7,8 +7,8 @@
         <img src="https://opencollective.com/shields/backers/badge.svg" /></a>
     <a href="#sponsors" alt="Sponsors on Open Collective">
         <img src="https://opencollective.com/shields/sponsors/badge.svg" /></a>
-    <a href="https://travis-ci.org/badges/shields">
-        <img src="https://img.shields.io/travis/badges/shields.svg"
+    <a href="https://circleci.com/gh/badges/shields/tree/master">
+        <img src="https://img.shields.io/circleci/project/github/badges/shields.svg"
             alt="build status"></a>
     <a href="https://github.com/badges/shields/commits/gh-pages">
         <img src="https://img.shields.io/github/last-commit/badges/shields/gh-pages.svg?label=last%20deployed"

package.json

@@ -43,20 +43,22 @@
     "lodash.uniq": "~4.5.0"
   },
   "scripts": {
-    "coverage:test:js": "nyc node_modules/mocha/bin/_mocha '*.spec.js' 'lib/*.spec.js'",
+    "coverage:test:js": "nyc node_modules/mocha/bin/_mocha '*.spec.js' 'lib/**/*.spec.js' 'service-tests/**/*.spec.js'",
     "coverage:test:services": "nyc node_modules/mocha/bin/_mocha --delay service-tests/runner/cli.js",
     "coverage:test": "rimraf .nyc_output coverage; npm run coverage:test:js; npm run coverage:test:services",
     "coverage:report": "nyc report",
     "coverage:report:reopen": "opn coverage/lcov-report/index.html",
     "coverage:report:open": "npm run coverage:report && npm run coverage:report:reopen",
     "lint": "eslint '**/*.js'",
     "test:js:frontend": "mocha --require babel-polyfill --require babel-register 'frontend/**/*.spec.js'",
-    "test:js:server": "mocha '*.spec.js' 'lib/**/*.spec.js'",
+    "test:js:server": "mocha '*.spec.js' 'lib/**/*.spec.js' 'service-tests/**/*.spec.js'",
     "test:services": "mocha --delay service-tests/runner/cli.js",
     "test:services:pr:prepare": "node service-tests/runner/pull-request-services-cli.js > pull-request-services.log",
     "test:services:pr:run": "mocha --delay service-tests/runner/cli.js --stdin < pull-request-services.log",
     "test:services:pr": "npm run test:services:pr:prepare && npm run test:services:pr:run",
     "test": "npm run lint && npm run test:js:frontend && npm run test:js:server",
+    "circle-images:build": "docker build -t shieldsio/shields-node-8:${IMAGE_TAG} -f .circleci/images/node-8/Dockerfile . && docker build -t shieldsio/shields-node-6:${IMAGE_TAG} -f .circleci/images/node-6/Dockerfile .",
+    "circle-images:push": "docker push shieldsio/shields-node-8:${IMAGE_TAG} && docker push shieldsio/shields-node-6:${IMAGE_TAG}",
     "frontend-depcheck": "check-node-version --node \">= 8.0\"",
     "server-depcheck": "check-node-version --node \">= 6.0 < 9.0\"",
     "postinstall": "npm run server-depcheck",

service-tests/runner/infer-pull-request.js

@@ -0,0 +1,75 @@
+'use strict';
+
+const { parse: urlParse, format: urlFormat } = require('url');
+
+function formatSlug(owner, repo, pullRequest) {
+  return `${owner}/${repo}#${pullRequest}`;
+}
+
+function parseGithubPullRequestUrl(url, options = {}) {
+  const { verifyBaseUrl } = options;
+
+  const parsed = urlParse(url);
+  const components = parsed.path.substr(1).split('/');
+  if (components[2] !== 'pull' || components.length !== 4) {
+    throw Error(`Invalid GitHub pull request URL: ${url}`);
+  }
+  const [owner, repo, , pullRequest] = components;
+
+  delete parsed.pathname;
+  const baseUrl = urlFormat(parsed, { auth: false, fragment: false, search: false });
+
+  if (verifyBaseUrl && baseUrl !== verifyBaseUrl) {
+    throw Error(`Expected base URL to be ${verifyBaseUrl} but got ${baseUrl}`);
+  }
+
+  return {
+    baseUrl,
+    owner,
+    repo,
+    pullRequest: +pullRequest,
+    slug: formatSlug(owner, repo, pullRequest),
+  };
+}
+
+function parseGithubRepoSlug(slug) {
+  const components = slug.split('/');
+  if (components.length !== 2) {
+    throw Error(`Invalid GitHub repo slug: ${slug}`);
+  }
+  const [owner, repo] = components;
+  return { owner, repo };
+}
+
+function _inferPullRequestFromTravisEnv(env) {
+  const { owner, repo } = parseGithubRepoSlug(env.TRAVIS_REPO_SLUG);
+  const pullRequest = +env.TRAVIS_PULL_REQUEST;
+  return {
+    owner,
+    repo,
+    pullRequest,
+    slug: formatSlug(owner, repo, pullRequest),
+  };
+}
+
+function _inferPullRequestFromCircleEnv(env) {
+  return parseGithubPullRequestUrl(env.CI_PULL_REQUEST);
+}
+
+function inferPullRequest(env = process.env) {
+  if (env.TRAVIS) {
+    return _inferPullRequestFromTravisEnv(env);
+  } else if (env.CIRCLECI) {
+    return _inferPullRequestFromCircleEnv(env);
+  } else if (env.CI) {
+    throw Error('Unsupported CI system. Unable to obtain pull request information from the environment.');
+  } else {
+    throw Error('Unable to obtain pull request information from the environment. Is this running in CI?');
+  }
+}
+
+module.exports = {
+  parseGithubPullRequestUrl,
+  parseGithubRepoSlug,
+  inferPullRequest,
+};