Build/deploy pipeline using GitHub actions

.github/workflows/build-and-trigger-deploy.yml

@@ -0,0 +1,72 @@
+name: Build and Trigger Deploy
+
+on:
+  push:
+    branches:
+      - master
+  create:
+    tags:
+      - 'v*'
+
+env:
+  IMAGEID: rapidlua/luajit.me
+  PACKERV: 1.5.1
+
+jobs:
+
+  cloud-images:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install packer
+      run: >
+        curl https://releases.hashicorp.com/packer/${PACKERV}/packer_${PACKERV}_linux_$(dpkg --print-architecture).zip > packer.zip &&
+        unzip packer.zip &&
+        sudo install packer /usr/bin &&
+        rm packer.zip packer
+    - uses: actions/checkout@v1
+    - name: Build cloud images
+      run: >
+        VERSION=$(git describe --tags | sed -e s/^v//);
+        packer build -var "version=${VERSION}" -var "digitalocean_token=${{ secrets.DIGITALOCEAN_TOKEN }}" deploy/cloud-images.json
+
+  docker-image-amd64:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Build Docker image
+      run: >
+        VERSION=$(git describe --tags | sed -e s/^v//);
+        mkdir -p build &&
+        (cd app && DOCKER_BUILDKIT=1 docker build . --tag "${IMAGEID}:${VERSION}-amd64") &&
+        docker save "${IMAGEID}:${VERSION}-amd64" | gzip > build/image.tar.gz &&
+        echo "$VERSION" > build/version
+    - uses: actions/upload-artifact@v1.0.0
+      with: { name: docker-image-amd64, path: build }
+
+  postprocess-and-trigger-deploy:
+    runs-on: ubuntu-latest
+    needs: [cloud-images, docker-image-amd64]
+    steps:
+      - uses: actions/download-artifact@v1.0.0
+        with: { name: docker-image-amd64 }
+      - name: Configure Docker client / login
+        run: >
+          mkdir -p ~/.docker &&
+          echo '{"experimental":"enabled"}' > ~/.docker/config.json &&
+          echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u mejedi --password-stdin
+      - name: Upload Docker images
+        run: >
+          VERSION=$(cat docker-image-amd64/version);
+          IID=${IMAGEID}:${VERSION};
+          zcat docker-image-amd64/image.tar.gz | docker image load &&
+          docker push "${IID}-amd64" &&
+          docker manifest create "${IID}" "${IID}-amd64" &&
+          docker manifest push "${IID}"
+      - name: Trigger deploy
+        run: >
+          if cat docker-image-amd64/version | grep -q -- -g; then
+            ENV=staging
+          else
+            ENV=production
+          fi;
+          curl -sd "{\"ref\":\"${GITHUB_SHA}\",\"required_contexts\":[],\"environment\":\"${ENV}\"}" https://api.github.com/repos/${GITHUB_REPOSITORY}/deployments -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}"

.github/workflows/gc-cloud-images.js

@@ -0,0 +1,132 @@
+const URL = require('url').URL;
+const https = require('https');
+const child_process = require('child_process');
+
+function apiCall(url, method, headers, cb) {
+  const u = new URL(url);
+  const req = https.request({ hostname: u.hostname, path: u.pathname + u.search, method, headers }, (res) => {
+    if (res.statusCode !== 200 && res.statusCode !== 204) {
+      cb(new Error(res.statusMessage));
+      return;
+    }
+    const chunks = [];
+    res.on('data', chunk => chunks.push(chunk));
+    res.on('end', () => {
+      try {
+        if (res.statusCode === 204) cb(null, null);
+        else cb(null, JSON.parse(Buffer.concat(chunks).toString()), res);
+      } catch (e) {
+        cb(e);
+      }
+    });
+  });
+  req.on('error', cb);
+  req.end();
+}
+
+function getDeployedHashes(headers, environments, cb) {
+  const hashes = [];
+  const envFound  = new Array(environments ? environments.length : 2).fill(false);
+  let pending = 1;
+  function fetch(url) {
+    apiCall(url, 'GET', headers, (err, responseJSON, response) => {
+      if (err) {
+        cb && cb(err);
+        cb = null;
+        return;
+      }
+      for (let deployment of responseJSON) {
+        const index = environments ? environments.indexOf(deployment.environment) : 0;
+        if (index >= 0 && deployment.statuses_url.startsWith('https://api.github.com/')) {
+          ++pending;
+          apiCall(deployment.statuses_url, 'GET', headers, (err, statuses) => {
+            if (err) {
+              cb && cb(err);
+              cb = null;
+              return;
+            }
+            if (statuses[0] && statuses[0].state == 'success') {
+              hashes.push(deployment.sha);
+              envFound[index] = true;
+            }
+            if (!--pending && cb) {
+              cb(null, hashes);
+              cb = null;
+            }
+          });
+        }
+      }
+      const match = (response.headers.link || '').match(/<(.[^>]+)>;\s*rel="next"/);
+      if (cb && match && match[1] && match[1].startsWith('https://api.github.com/')) {
+        if (!envFound.every(_ => _)) {
+          fetch(match[1]);
+          return;
+        }
+      }
+      if (!--pending && cb) {
+        cb(null, hashes);
+        cb = null;
+      }
+    });
+  }
+  fetch('https://api.github.com/repos/rapidlua/luajit.me/deployments');
+}
+
+function listDigitalOceanPrivateImages(headers, cb) {
+  const images = [];
+  function fetch(url) {
+    apiCall(url, 'GET', headers, (err, response) => {
+      if (err)
+        return cb(err);
+      images.push(...response.images);
+      try {
+        if (response.links.pages.next.startsWith('https://api.digitalocean.com/'))
+          return fetch(response.links.pages.next);
+      } catch (e) {}
+      cb(null, images);
+    });
+  }
+  fetch('https://api.digitalocean.com/v2/images?private=true&per_page=200');
+}
+
+function isOrphanedImage(name, hashes) {
+  if (!name.startsWith('image-'))
+    return false;
+  const res = child_process.spawnSync(
+    'git', [ 'rev-parse', '--verify', 'v' + name.substr(6) ],
+    { encoding: 'utf8' }
+  );
+  return res.status || !hashes.includes(res.stdout.trim());
+}
+
+function fatal(...args) {
+  console.error(...args);
+  process.exit(1);
+}
+
+const githubHeaders = {
+  'User-Agent': 'rapidlua',
+  Authorization: 'token ' + process.env.GITHUB_TOKEN,
+  Accept: 'application/vnd.github.ant-man-preview+json'
+};
+
+const digitalOceanHeaders = {
+  Authorization: 'Bearer ' + process.env.DIGITALOCEAN_TOKEN
+}
+
+function hourToMSec(v) { return v * 1000 * 60 * 60; }
+
+getDeployedHashes(githubHeaders, [ 'production' ], (err, hashes) => {
+  if (err) fatal('get deployed hashes:', err);
+  listDigitalOceanPrivateImages(digitalOceanHeaders, (err, images) => {
+    if (err) fatal('list DigitalOcean private images:', err);
+    for (let image of images) {
+      if (image.type === 'snapshot' && Date.now() - new Date(image.created_at) > hourToMSec(3) && isOrphanedImage(image.name, hashes)) {
+        apiCall('https://api.digitalocean.com/v2/images/' + image.id, 'DELETE', digitalOceanHeaders, (err) => {
+          if (err) fatal('removing ' + image.name + ':', err);
+          console.log('removed', image.name);
+        });
+      }
+    }
+  });
+});

.github/workflows/gc-cloud-images.yml

@@ -0,0 +1,14 @@
+name: GC Cloud Images
+
+on:
+  schedule:
+    - cron: '* */3 * * *'
+
+jobs:
+  gc-cloud-images:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - run: >
+        GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN}} DIGITALOCEAN_TOKEN=${{ secrets.DIGITALOCEAN_TOKEN }} node .github/workflows/gc-cloud-images.js
+

deploy/cloud-images.json

@@ -0,0 +1,39 @@
+{
+  "variables": {
+    "version": "",
+    "digitalocean_token": ""
+  },
+  "builders": [
+    {
+      "type": "digitalocean",
+      "api_token": "{{user `digitalocean_token`}}",
+      "image": "ubuntu-18-04-x64",
+      "region": "ams3",
+      "size": "s-1vcpu-1gb",
+      "snapshot_name": "image-{{user `version`}}",
+      "ssh_username": "root"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "apt-get update",
+        "DEBIAN_FRONTEND=noninteractive apt-get upgrade -y --no-install-recommends",
+
+        "apt-get install -y --no-install-recommends apt-transport-https curl",
+        "apt-get install -y --no-install-recommends ca-certificates gnupg-agent software-properties-common",
+        "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -",
+        "add-apt-repository \"deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\"",
+        "apt-get update",
+        "mkdir -p /etc/docker",
+        "echo '{\"userns-remap\":\"default\"}' > /etc/docker/daemon.json",
+        "apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io",
+        "docker pull node:10-alpine",
+
+        "apt-get install -y --no-install-recommends nginx",
+        "systemctl disable nginx"
+      ]
+    }
+  ]
+}